1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
commit a81f05034baef438eacd346fcbc97dd818a0b5bf
Author: Vishesh Handa <me@vhanda.in>
Date: Mon Aug 6 15:01:14 2012 +0530
kinotify: Do not store the paths to be added
kinotify used a QQueue<QByteArray> to store all the directories which
need to be added. Since we use QDirIterator, each path is presented as a
QString which is then encoded to its QByteArray (QFile::encodeName).
This results in *large* chunks of memory being allocated, and then
slowly being deallocated.
Instead, we now use a QDirIterator, and do not store all the directories
which need to be added, we simply iterate over them. This way we do not
allocate large amounts of memory.
There is a large performance improvement as well. On my system, with
38829 directories, adding all the watches now takes only about 10 seconds,
instead of about 65.
Patch possible due to massif output provided by Jure Repinc <jlp@holodeck1.com>.
Thanks a lot.
tldr: Use DFS instead of BFS -> Less memory consumption
BUG: 304476
REVIEW: 105892
DIGEST: Fix massive memory leak in Nepomuk File Monitoring Service
diff --git a/services/filewatch/kinotify.cpp b/services/filewatch/kinotify.cpp
index e8843c8..47eb8ed 100644
--- a/services/filewatch/kinotify.cpp
+++ b/services/filewatch/kinotify.cpp
@@ -79,8 +79,8 @@ public:
QHash<int, QByteArray> watchPathHash;
QHash<QByteArray, int> pathWatchHash;
- /// queue of paths to install watches for
- QQueue<QByteArray> pathsToWatch;
+ /// A list of all the current dirIterators
+ QQueue<QDirIterator*> dirIterators;
unsigned char eventBuffer[EVENT_BUFFER_SIZE];
@@ -136,20 +136,6 @@ public:
}
}
- bool addWatchesRecursively( const QByteArray& path )
- {
- if ( !addWatch( path ) )
- return false;
-
- const QString stringPath = QFile::decodeName(path);
- QDirIterator iter( stringPath, QDir::Dirs | QDir::NoDotAndDotDot );
- while( iter.hasNext() ) {
- pathsToWatch.enqueue( QFile::encodeName(iter.next()) );
- }
-
- return true;
- }
-
void removeWatch( int wd ) {
kDebug() << wd << watchPathHash[wd];
pathWatchHash.remove( watchPathHash.take( wd ) );
@@ -159,19 +145,20 @@ public:
void _k_addWatches() {
// add the next batch of paths
for ( int i = 0; i < 100; ++i ) {
- if ( pathsToWatch.isEmpty() ||
- !addWatchesRecursively( pathsToWatch.dequeue() ) ) {
- return;
+ QDirIterator* it = dirIterators.front();
+ if( it->hasNext() ) {
+ it->next();
+ addWatch( QFile::encodeName(it->filePath()) );
+ }
+ else {
+ delete dirIterators.dequeue();
}
}
// asyncroneously add the next batch
- if ( !pathsToWatch.isEmpty() ) {
+ if ( !dirIterators.isEmpty() ) {
QMetaObject::invokeMethod( q, "_k_addWatches", Qt::QueuedConnection );
}
- else {
- kDebug() << "All watches installed";
- }
}
private:
@@ -245,7 +232,10 @@ bool KInotify::addWatch( const QString& path, WatchEvents mode, WatchFlags flags
d->mode = mode;
d->flags = flags;
- d->pathsToWatch.append( QFile::encodeName( path ) );
+ d->addWatch( QFile::encodeName(path) );
+ QDirIterator* iter = new QDirIterator( path, QDir::Dirs | QDir::NoDotAndDotDot,
+ QDirIterator::Subdirectories );
+ d->dirIterators.append( iter );
d->_k_addWatches();
return true;
}
|
GitWeb