fanotify启用时，多线程打开文件会挂起 [英] multi-thread opening file hangs when fanotify is on
问题描述
我使用这个fanotify示例来监视整个文件系统(/)上的打开/访问权限: http://git.infradead.org/users/eparis/fanotify-example.git .
I use this fanotify sample to monitor open/access perms on the whole file system(/): http://git.infradead.org/users/eparis/fanotify-example.git.
然后我有一个具有多个线程的测试程序，每个线程都会迭代样本查找程序并打开/关闭其中的文件，有时我的程序会挂在open()上.
Then I have a test program with multiple threads, each thread iterate the sample foder and open/close the files in it, sometimes my program hangs at open().
OS:Ubuntu 2.6.38-11 x86_64.
OS: Ubuntu 2.6.38-11 x86_64.
不支持多线程打开是幻想的错误吗?
Is it a bug of fanotify that it does not support multiple-thread opening?
我的测试程序的代码:
#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>
#include <ctype.h>
#include <unistd.h>
#include <dirent.h>
#include <string.h>
#include <sys/stat.h>
#include <errno.h>
#include <fcntl.h>
#include <dirent.h>
//open file function
void open_file( char* file )
{
int fd = -1;
fd = open( file, O_WRONLY, 0x666 );
if( fd >= 0 )
{
printf("open:%s\n", file );
close( fd );
}
}
//iterate directory function
void printdir(char *dir, int depth)
{
DIR *dp;
struct stat statbuf;
char pathbuf[2048] = {0};
struct dirent entry;
struct dirent *entryPtr = NULL;
//printf("opendir %s\n", dir );
usleep( 300 );
if((dp = opendir(dir)) == NULL) {
if( errno != ENOTDIR )
{
fprintf(stderr,"cannot open directory: %s\n", dir);
perror("open fial");
}
return;
}
readdir_r( dp, &entry, &entryPtr );
while( entryPtr != NULL)
{
snprintf(pathbuf,2000, "%s/%s\0", dir, entry.d_name );
printf("iteraotr:%s\n", pathbuf );
lstat( pathbuf, &statbuf );
if(S_ISDIR( statbuf.st_mode ))
{
/* Found a directory, but ignore . and .. */
if(strcmp(".",entry.d_name) == 0 ||
strcmp("..",entry.d_name) == 0)
{
}
else
{
//printf("%d,%s\n",depth, entry->d_name);
printdir( pathbuf, depth+1);
}
}
else
{
//printf("%*s%s\n",depth,"",entry->d_name);
open_file( pathbuf );
}
readdir_r( dp, &entry, &entryPtr );
}
closedir(dp);
}
//thread function
void* iterator_dir( void* data )
{
char* path = (char*)data;
printf("In iterator_dir(): %s\n", path );
printdir( path, 0 );
return NULL;
}
pthread_t threadID[10] = {0};
//main function
int main( int argc, char** argv )
{
if( argc < 3 )
{
printf("Usage: %s <thread_num> <file>\n", argv[0] );
exit(0);
}
if( isdigit( (char)*argv[1] ) == 0 )
{
printf(" Thread num is 0 - 9\n");
exit(0);
}
int thread_num = atoi( argv[1] );
char* res;
pthread_attr_t attr;
pthread_attr_init(&attr);
int i = 0;
for( i = 0; i < thread_num; ++i )
{
pthread_create( &threadID[i], &attr, &iterator_dir, argv[2]);
}
for( i = 0; i < thread_num; ++i )
{
pthread_join( threadID[i] , &res );
}
}
2011-09-28
我评论了打开文件的操作，只保留了迭代目录的一部分.该应用程序仍然挂起. 这是strace的输出:
I comment the open file operation, only keep the iterate directory part. The application still hangs. This is the output of strace:
enter code here
pid 10692] open("/home/byang//.config/menus", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC <unfinished ...>
[pid 10691] write(1, "1213966080 opendir /home/byang//"..., 56) = 56
.........
.........
[pid 10689] madvise(0x7f3c48dbc000, 8368128, MADV_DONTNEED) = 0
[pid 10689] _exit(0) = ?
Process 10689 detached
[pid 10688] <... futex resumed> ) = 0
[pid 10688] futex(0x7f3c47db99d0, FUTEX_WAIT, 10692, NULL <unfinished ...>
它挂在这里，当我关闭fanotify时，它继续...
It hangs here, when I close the fanotify, it continues...
[pid 10692] <... open resumed> ) = 11
[pid 10692] getdents(11, /* 4 entries */, 32768) = 128
[pid 10692] lstat("/home/byang//.config/menus/applications.menu", {st_mode=S_IFREG|0644, st_size=233, ...}) = 0
10688是父线程； 10689,10691,10692是迭代目录的子线程. 似乎10692正在等待fanotify的回复?
10688 is the parent thread; 10689,10691,10692 are the child threads that iterating the directories. It seems that 10692 are waiting the reply from fanotify?
推荐答案
这是内核的幻想的错误. 我向Linux内核发布了一个补丁: 当多个线程位于同一个目录中时，某些线程将挂起. 该补丁可以使不同的访问事件区别开来 线程，防止fanotify合并来自不同站点的访问事件 线程.
It's a bug of Kernel's fanotify. I posted a patch to Linux-Kernel: When multiple threadsiterate the same direcotry, some thread will hang. This patch let fanotify differentiate access events from different threads, prevent fanotify from merging access events from different threads.
http://marc.info/?l=linux-内核& m = 131822913806350& w = 2
-------------------------------
diff -r -u linux-3.1-rc4_orig/fs/notify/fanotify/fanotify.c
linux-3.1-rc4/fs/notify/fanotify/fanotify.c
--- linux-3.1-rc4_orig/fs/notify/fanotify/fanotify.c 2011-08-29
12:16:01.000000000 +0800
+++ linux-3.1-rc4/fs/notify/fanotify/fanotify.c 2011-10-10
12:28:23.276847000 +0800
@@ -15,7 +15,8 @@
if (old->to_tell == new->to_tell &&
old->data_type == new->data_type &&
- old->tgid == new->tgid) {
+ old->tgid == new->tgid &&
+ old->pid == new->pid) {
switch (old->data_type) {
case (FSNOTIFY_EVENT_PATH):
if ((old->path.mnt == new->path.mnt) &&
@@ -144,11 +145,19 @@
return PTR_ERR(notify_event);
#ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
- if (event->mask & FAN_ALL_PERM_EVENTS) {
- /* if we merged we need to wait on the new event */
- if (notify_event)
- event = notify_event;
- ret = fanotify_get_response_from_access(group, event);
+ //if overflow, do not wait for response
+ if(fsnotify_isoverflow(event))
+ {
+ pr_debug("fanotify overflow!\n");
+ }
+ else
+ {
+ if (event->mask & FAN_ALL_PERM_EVENTS) {
+ /* if we merged we need to wait on the new event */
+ if (notify_event)
+ event = notify_event;
+ ret = fanotify_get_response_from_access(group, event);
+ }
}
#endif
diff -r -u linux-3.1-rc4_orig/fs/notify/notification.c
linux-3.1-rc4/fs/notify/notification.c
--- linux-3.1-rc4_orig/fs/notify/notification.c 2011-08-29
12:16:01.000000000 +0800
+++ linux-3.1-rc4/fs/notify/notification.c 2011-10-10 12:27:09.331787000 +0800
@@ -95,6 +95,7 @@
BUG_ON(!list_empty(&event->private_data_list));
kfree(event->file_name);
+ put_pid(event->pid);
put_pid(event->tgid);
kmem_cache_free(fsnotify_event_cachep, event);
}
@@ -132,6 +133,14 @@
return priv;
}
+bool fsnotify_isoverflow(struct fsnotify_event *event)
+{
+ if(event==q_overflow_event)
+ {
+ return true;
+ }
+ return false;
+}
/*
* Add an event to the group notification queue. The group can later pull this
* event off the queue to deal with. If the event is successfully added to the
@@ -374,6 +383,7 @@
return NULL;
}
}
+ event->pid = get_pid(old_event->pid);
event->tgid = get_pid(old_event->tgid);
if (event->data_type == FSNOTIFY_EVENT_PATH)
path_get(&event->path);
@@ -417,6 +427,7 @@
event->name_len = strlen(event->file_name);
}
+ event->pid = get_pid(task_pid(current));
event->tgid = get_pid(task_tgid(current));
event->sync_cookie = cookie;
event->to_tell = to_tell;
diff -r -u linux-3.1-rc4_orig/include/linux/fsnotify_backend.h
linux-3.1-rc4/include/linux/fsnotify_backend.h
--- linux-3.1-rc4_orig/include/linux/fsnotify_backend.h 2011-08-29
12:16:01.000000000 +0800
+++ linux-3.1-rc4/include/linux/fsnotify_backend.h 2011-10-10
12:27:48.587369000 +0800
@@ -238,6 +238,7 @@
u32 sync_cookie; /* used to corrolate events, namely inotify mv events */
const unsigned char *file_name;
size_t name_len;
+ struct pid *pid;
struct pid *tgid;
#ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS
@@ -378,6 +379,8 @@
struct fsnotify_event_private_data *priv,
struct fsnotify_event *(*merge)(struct list_head *,
struct fsnotify_event *));
+/*true if the event is an overflow event*/
+extern bool fsnotify_isoverflow(struct fsnotify_event *event);
/* true if the group notification queue is empty */
extern bool fsnotify_notify_queue_is_empty(struct fsnotify_group *group);
/* return, but do not dequeue the first event on the notification queue */
这篇关于fanotify启用时，多线程打开文件会挂起的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！
