No issues found
| Tool | Failure ID | Location | Function | Message | Data |
|---|---|---|---|---|---|
| clang-analyzer | no-output-found | tracker-miner-fs.c | Message(text='Unable to locate XML output from invoke-clang-analyzer') | None |
1 /*
2 * Copyright (C) 2009, Nokia <ivan.frade@nokia.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
18 */
19
20 #include "config.h"
21
22 #include <libtracker-common/tracker-date-time.h>
23 #include <libtracker-common/tracker-dbus.h>
24 #include <libtracker-common/tracker-file-utils.h>
25 #include <libtracker-common/tracker-log.h>
26 #include <libtracker-common/tracker-utils.h>
27
28 #include "tracker-crawler.h"
29 #include "tracker-marshal.h"
30 #include "tracker-miner-fs.h"
31 #include "tracker-media-art.h"
32 #include "tracker-monitor.h"
33 #include "tracker-utils.h"
34 #include "tracker-thumbnailer.h"
35 #include "tracker-priority-queue.h"
36 #include "tracker-task-pool.h"
37 #include "tracker-sparql-buffer.h"
38 #include "tracker-file-notifier.h"
39
40 /* If defined will print the tree from GNode while running */
41 #ifdef CRAWLED_TREE_ENABLE_TRACE
42 #warning Tree debugging traces enabled
43 #endif /* CRAWLED_TREE_ENABLE_TRACE */
44
45 /* If defined will print push/pop actions on queues */
46 #ifdef EVENT_QUEUE_ENABLE_TRACE
47 #warning Event Queue traces enabled
48 #define EVENT_QUEUE_LOG_PREFIX "[Event Queues] "
49 #define EVENT_QUEUE_STATUS_TIMEOUT_SECS 30
50 #define trace_eq(message, ...) g_debug (EVENT_QUEUE_LOG_PREFIX message, ##__VA_ARGS__)
51 #define trace_eq_action(pushed, queue_name, position, gfile1, gfile2, reason) \
52 do { \
53 gchar *uri1 = g_file_get_uri (gfile1); \
54 gchar *uri2 = gfile2 ? g_file_get_uri (gfile2) : NULL; \
55 g_debug ("%s%s '%s%s%s' %s %s of queue '%s'%s%s", \
56 EVENT_QUEUE_LOG_PREFIX, \
57 pushed ? "Pushed" : "Popped", \
58 uri1, \
59 uri2 ? "->" : "", \
60 uri2 ? uri2 : "", \
61 pushed ? "to" : "from", \
62 position, \
63 queue_name, \
64 reason ? ": " : "", \
65 reason ? reason : ""); \
66 g_free (uri1); \
67 g_free (uri2); \
68 } while (0)
69 #define trace_eq_push_tail(queue_name, gfile, reason) \
70 trace_eq_action (TRUE, queue_name, "tail", gfile, NULL, reason)
71 #define trace_eq_push_head(queue_name, gfile, reason) \
72 trace_eq_action (TRUE, queue_name, "head", gfile, NULL, reason)
73 #define trace_eq_push_tail_2(queue_name, gfile1, gfile2, reason) \
74 trace_eq_action (TRUE, queue_name, "tail", gfile1, gfile2, reason)
75 #define trace_eq_push_head_2(queue_name, gfile1, gfile2, reason) \
76 trace_eq_action (TRUE, queue_name, "head", gfile1, gfile2, reason)
77 #define trace_eq_pop_head(queue_name, gfile) \
78 trace_eq_action (FALSE, queue_name, "head", gfile, NULL, NULL)
79 #define trace_eq_pop_head_2(queue_name, gfile1, gfile2) \
80 trace_eq_action (FALSE, queue_name, "head", gfile1, gfile2, NULL)
81 static gboolean miner_fs_queues_status_trace_timeout_cb (gpointer data);
82 #else
83 #define trace_eq(...)
84 #define trace_eq_push_tail(...)
85 #define trace_eq_push_head(...)
86 #define trace_eq_push_tail_2(...)
87 #define trace_eq_push_head_2(...)
88 #define trace_eq_pop_head(...)
89 #define trace_eq_pop_head_2(...)
90 #endif /* EVENT_QUEUE_ENABLE_TRACE */
91
92 /* Number of times a GFile can be re-queued before it's dropped for
93 * whatever reason to avoid infinite loops.
94 */
95 #define REENTRY_MAX 2
96
97 /* Default processing pool limits to be set */
98 #define DEFAULT_WAIT_POOL_LIMIT 1
99 #define DEFAULT_READY_POOL_LIMIT 1
100
101 /* Put tasks processing at a lower priority so other events
102 * (timeouts, monitor events, etc...) are guaranteed to be
103 * dispatched promptly.
104 */
105 #define TRACKER_TASK_PRIORITY G_PRIORITY_DEFAULT_IDLE + 10
106
107 /**
108 * SECTION:tracker-miner-fs
109 * @short_description: Abstract base class for filesystem miners
110 * @include: libtracker-miner/tracker-miner.h
111 *
112 * #TrackerMinerFS is an abstract base class for miners that collect data
113 * from the filesystem, all the filesystem crawling and monitoring is
114 * abstracted away, leaving to implementations the decisions of what
115 * directories/files should it process, and the actual data extraction.
116 **/
117
118 #define TRACKER_MINER_FS_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_MINER_FS, TrackerMinerFSPrivate))
119
120 typedef struct {
121 GFile *file;
122 GFile *source_file;
123 } ItemMovedData;
124
125 typedef struct {
126 GFile *file;
127 GPtrArray *results;
128 GStrv rdf_types;
129 GCancellable *cancellable;
130 guint notified : 1;
131 } ItemWritebackData;
132
133 typedef struct {
134 GFile *file;
135 gchar *urn;
136 gchar *parent_urn;
137 gint priority;
138 GCancellable *cancellable;
139 TrackerSparqlBuilder *builder;
140 TrackerMiner *miner;
141 } UpdateProcessingTaskContext;
142
143 typedef struct {
144 GMainLoop *main_loop;
145 GString *sparql;
146 const gchar *source_uri;
147 const gchar *uri;
148 } RecursiveMoveData;
149
150 struct _TrackerMinerFSPrivate {
151 /* File queues for indexer */
152 TrackerPriorityQueue *items_created;
153 TrackerPriorityQueue *items_updated;
154 TrackerPriorityQueue *items_deleted;
155 TrackerPriorityQueue *items_moved;
156 TrackerPriorityQueue *items_writeback;
157 #ifdef EVENT_QUEUE_ENABLE_TRACE
158 guint queue_status_timeout_id;
159 #endif /* EVENT_QUEUE_ENABLE_TRACE */
160
161 TrackerFileNotifier *file_notifier;
162
163 GHashTable *items_ignore_next_update;
164
165 GQuark quark_ignore_file;
166 GQuark quark_attribute_updated;
167 GQuark quark_directory_found_crawling;
168 GQuark quark_reentry_counter;
169
170 GTimer *timer;
171 GTimer *extraction_timer;
172
173 guint item_queues_handler_id;
174 GFile *item_queue_blocker;
175
176 gdouble throttle;
177
178 /* Extraction tasks */
179 TrackerTaskPool *task_pool;
180
181 /* Writeback tasks */
182 TrackerTaskPool *writeback_pool;
183
184 /* Sparql insertion tasks */
185 TrackerSparqlBuffer *sparql_buffer;
186 guint sparql_buffer_limit;
187
188 TrackerIndexingTree *indexing_tree;
189
190 /* Status */
191 guint been_started : 1; /* TRUE if miner has been started */
192 guint been_crawled : 1; /* TRUE if initial crawling has been
193 * done */
194 guint shown_totals : 1; /* TRUE if totals have been shown */
195 guint is_paused : 1; /* TRUE if miner is paused */
196 guint mtime_checking : 1; /* TRUE if mtime checks should be done
197 * during initial crawling. */
198 guint initial_crawling : 1; /* TRUE if initial crawling should be
199 * done */
200 guint timer_stopped : 1; /* TRUE if main timer is stopped */
201 guint extraction_timer_stopped : 1; /* TRUE if the extraction
202 * timer is stopped */
203
204 /* Statistics */
205 guint total_directories_found;
206 guint total_directories_ignored;
207 guint total_files_found;
208 guint total_files_ignored;
209
210 guint directories_found;
211 guint directories_ignored;
212 guint files_found;
213 guint files_ignored;
214
215 guint total_files_processed;
216 guint total_files_notified;
217 guint total_files_notified_error;
218 };
219
220 typedef enum {
221 QUEUE_NONE,
222 QUEUE_CREATED,
223 QUEUE_UPDATED,
224 QUEUE_DELETED,
225 QUEUE_MOVED,
226 QUEUE_IGNORE_NEXT_UPDATE,
227 QUEUE_WAIT,
228 QUEUE_WRITEBACK
229 } QueueState;
230
231 enum {
232 PROCESS_FILE,
233 PROCESS_FILE_ATTRIBUTES,
234 IGNORE_NEXT_UPDATE_FILE,
235 FINISHED,
236 WRITEBACK_FILE,
237 LAST_SIGNAL
238 };
239
240 enum {
241 PROP_0,
242 PROP_THROTTLE,
243 PROP_WAIT_POOL_LIMIT,
244 PROP_READY_POOL_LIMIT,
245 PROP_MTIME_CHECKING,
246 PROP_INITIAL_CRAWLING
247 };
248
249 static void miner_fs_initable_iface_init (GInitableIface *iface);
250
251 static void fs_finalize (GObject *object);
252 static void fs_set_property (GObject *object,
253 guint prop_id,
254 const GValue *value,
255 GParamSpec *pspec);
256 static void fs_get_property (GObject *object,
257 guint prop_id,
258 GValue *value,
259 GParamSpec *pspec);
260 static void miner_started (TrackerMiner *miner);
261 static void miner_stopped (TrackerMiner *miner);
262 static void miner_paused (TrackerMiner *miner);
263 static void miner_resumed (TrackerMiner *miner);
264 static void miner_ignore_next_update (TrackerMiner *miner,
265 const GStrv subjects);
266 static ItemMovedData *item_moved_data_new (GFile *file,
267 GFile *source_file);
268 static void item_moved_data_free (ItemMovedData *data);
269 static void item_writeback_data_free (ItemWritebackData *data);
270
271 static void indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree,
272 GFile *directory,
273 gpointer user_data);
274 static void file_notifier_file_created (TrackerFileNotifier *notifier,
275 GFile *file,
276 gpointer user_data);
277 static void file_notifier_file_deleted (TrackerFileNotifier *notifier,
278 GFile *file,
279 gpointer user_data);
280 static void file_notifier_file_updated (TrackerFileNotifier *notifier,
281 GFile *file,
282 gboolean attributes_only,
283 gpointer user_data);
284 static void file_notifier_file_moved (TrackerFileNotifier *notifier,
285 GFile *source,
286 GFile *dest,
287 gpointer user_data);
288 static void file_notifier_directory_started (TrackerFileNotifier *notifier,
289 GFile *directory,
290 gpointer user_data);
291 static void file_notifier_directory_finished (TrackerFileNotifier *notifier,
292 GFile *directory,
293 guint directories_found,
294 guint directories_ignored,
295 guint files_found,
296 guint files_ignored,
297 gpointer user_data);
298 static void file_notifier_finished (TrackerFileNotifier *notifier,
299 gpointer user_data);
300
301 static void item_queue_handlers_set_up (TrackerMinerFS *fs);
302 static void item_update_children_uri (TrackerMinerFS *fs,
303 RecursiveMoveData *data,
304 const gchar *source_uri,
305 const gchar *uri);
306
307 static void task_pool_cancel_foreach (gpointer data,
308 gpointer user_data);
309 static void task_pool_limit_reached_notify_cb (GObject *object,
310 GParamSpec *pspec,
311 gpointer user_data);
312
313 static GInitableIface* miner_fs_initable_parent_iface;
314 static guint signals[LAST_SIGNAL] = { 0, };
315
316 G_DEFINE_ABSTRACT_TYPE_WITH_CODE (TrackerMinerFS, tracker_miner_fs, TRACKER_TYPE_MINER,
317 G_IMPLEMENT_INTERFACE (G_TYPE_INITABLE,
318 miner_fs_initable_iface_init));
319
320 static void
321 tracker_miner_fs_class_init (TrackerMinerFSClass *klass)
322 {
323 GObjectClass *object_class = G_OBJECT_CLASS (klass);
324 TrackerMinerClass *miner_class = TRACKER_MINER_CLASS (klass);
325
326 object_class->finalize = fs_finalize;
327 object_class->set_property = fs_set_property;
328 object_class->get_property = fs_get_property;
329
330 miner_class->started = miner_started;
331 miner_class->stopped = miner_stopped;
332 miner_class->paused = miner_paused;
333 miner_class->resumed = miner_resumed;
334 miner_class->ignore_next_update = miner_ignore_next_update;
335
336 g_object_class_install_property (object_class,
337 PROP_THROTTLE,
338 g_param_spec_double ("throttle",
339 "Throttle",
340 "Modifier for the indexing speed, 0 is max speed",
341 0, 1, 0,
342 G_PARAM_READWRITE));
343 g_object_class_install_property (object_class,
344 PROP_WAIT_POOL_LIMIT,
345 g_param_spec_uint ("processing-pool-wait-limit",
346 "Processing pool limit for WAIT tasks",
347 "Maximum number of files that can be concurrently "
348 "processed by the upper layer",
349 1, G_MAXUINT, DEFAULT_WAIT_POOL_LIMIT,
350 G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
351 g_object_class_install_property (object_class,
352 PROP_READY_POOL_LIMIT,
353 g_param_spec_uint ("processing-pool-ready-limit",
354 "Processing pool limit for READY tasks",
355 "Maximum number of SPARQL updates that can be merged "
356 "in a single connection to the store",
357 1, G_MAXUINT, DEFAULT_READY_POOL_LIMIT,
358 G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
359 g_object_class_install_property (object_class,
360 PROP_MTIME_CHECKING,
361 g_param_spec_boolean ("mtime-checking",
362 "Mtime checking",
363 "Whether to perform mtime checks during initial crawling or not",
364 TRUE,
365 G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
366 g_object_class_install_property (object_class,
367 PROP_INITIAL_CRAWLING,
368 g_param_spec_boolean ("initial-crawling",
369 "Initial crawling",
370 "Whether to perform initial crawling or not",
371 TRUE,
372 G_PARAM_READWRITE));
373
374 /**
375 * TrackerMinerFS::process-file:
376 * @miner_fs: the #TrackerMinerFS
377 * @file: a #GFile
378 * @builder: a #TrackerSparqlBuilder
379 * @cancellable: a #GCancellable
380 *
381 * The ::process-file signal is emitted whenever a file should
382 * be processed, and it's metadata extracted.
383 *
384 * @builder is the #TrackerSparqlBuilder where all sparql updates
385 * to be performed for @file will be appended.
386 *
387 * This signal allows both synchronous and asynchronous extraction,
388 * in the synchronous case @cancellable can be safely ignored. In
389 * either case, on successful metadata extraction, implementations
390 * must call tracker_miner_fs_file_notify() to indicate that
391 * processing has finished on @file, so the miner can execute
392 * the SPARQL updates and continue processing other files.
393 *
394 * Returns: %TRUE if the file is accepted for processing,
395 * %FALSE if the file should be ignored.
396 *
397 * Since: 0.8
398 **/
399 signals[PROCESS_FILE] =
400 g_signal_new ("process-file",
401 G_OBJECT_CLASS_TYPE (object_class),
402 G_SIGNAL_RUN_LAST,
403 G_STRUCT_OFFSET (TrackerMinerFSClass, process_file),
404 NULL, NULL,
405 tracker_marshal_BOOLEAN__OBJECT_OBJECT_OBJECT,
406 G_TYPE_BOOLEAN,
407 3, G_TYPE_FILE, TRACKER_SPARQL_TYPE_BUILDER, G_TYPE_CANCELLABLE);
408
409 /**
410 * TrackerMinerFS::process-file-attributes:
411 * @miner_fs: the #TrackerMinerFS
412 * @file: a #GFile
413 * @builder: a #TrackerSparqlBuilder
414 * @cancellable: a #GCancellable
415 *
416 * The ::process-file-attributes signal is emitted whenever a file should
417 * be processed, but only the attribute-related metadata extracted.
418 *
419 * @builder is the #TrackerSparqlBuilder where all sparql updates
420 * to be performed for @file will be appended. For the properties being
421 * updated, the DELETE statements should be included as well.
422 *
423 * This signal allows both synchronous and asynchronous extraction,
424 * in the synchronous case @cancellable can be safely ignored. In
425 * either case, on successful metadata extraction, implementations
426 * must call tracker_miner_fs_file_notify() to indicate that
427 * processing has finished on @file, so the miner can execute
428 * the SPARQL updates and continue processing other files.
429 *
430 * Returns: %TRUE if the file is accepted for processing,
431 * %FALSE if the file should be ignored.
432 *
433 * Since: 0.10
434 **/
435 signals[PROCESS_FILE_ATTRIBUTES] =
436 g_signal_new ("process-file-attributes",
437 G_OBJECT_CLASS_TYPE (object_class),
438 G_SIGNAL_RUN_LAST,
439 G_STRUCT_OFFSET (TrackerMinerFSClass, process_file_attributes),
440 NULL, NULL,
441 tracker_marshal_BOOLEAN__OBJECT_OBJECT_OBJECT,
442 G_TYPE_BOOLEAN,
443 3, G_TYPE_FILE, TRACKER_SPARQL_TYPE_BUILDER, G_TYPE_CANCELLABLE);
444
445 /**
446 * TrackerMinerFS::ignore-next-update-file:
447 * @miner_fs: the #TrackerMinerFS
448 * @file: a #GFile
449 * @builder: a #TrackerSparqlBuilder
450 * @cancellable: a #GCancellable
451 *
452 * The ::ignore-next-update-file signal is emitted whenever a file should
453 * be marked as to ignore on next update, and it's metadata prepared for that.
454 *
455 * @builder is the #TrackerSparqlBuilder where all sparql updates
456 * to be performed for @file will be appended.
457 *
458 * Returns: %TRUE on success
459 * %FALSE on failure
460 *
461 * Since: 0.8
462 *
463 * Deprecated: 0.12
464 **/
465 signals[IGNORE_NEXT_UPDATE_FILE] =
466 g_signal_new ("ignore-next-update-file",
467 G_OBJECT_CLASS_TYPE (object_class),
468 G_SIGNAL_RUN_LAST,
469 G_STRUCT_OFFSET (TrackerMinerFSClass, ignore_next_update_file),
470 NULL, NULL,
471 tracker_marshal_BOOLEAN__OBJECT_OBJECT_OBJECT,
472 G_TYPE_BOOLEAN,
473 3, G_TYPE_FILE, TRACKER_SPARQL_TYPE_BUILDER, G_TYPE_CANCELLABLE);
474
475 /**
476 * TrackerMinerFS::finished:
477 * @miner_fs: the #TrackerMinerFS
478 * @elapsed: elapsed time since mining was started
479 * @directories_found: number of directories found
480 * @directories_ignored: number of ignored directories
481 * @files_found: number of files found
482 * @files_ignored: number of ignored files
483 *
484 * The ::finished signal is emitted when @miner_fs has finished
485 * all pending processing.
486 *
487 * Since: 0.8
488 **/
489 signals[FINISHED] =
490 g_signal_new ("finished",
491 G_TYPE_FROM_CLASS (object_class),
492 G_SIGNAL_RUN_LAST,
493 G_STRUCT_OFFSET (TrackerMinerFSClass, finished),
494 NULL, NULL,
495 tracker_marshal_VOID__DOUBLE_UINT_UINT_UINT_UINT,
496 G_TYPE_NONE,
497 5,
498 G_TYPE_DOUBLE,
499 G_TYPE_UINT,
500 G_TYPE_UINT,
501 G_TYPE_UINT,
502 G_TYPE_UINT);
503
504 /**
505 * TrackerMinerFS::writeback-file:
506 * @miner_fs: the #TrackerMinerFS
507 * @file: a #GFile
508 * @rdf_types: the set of RDF types
509 * @results: (element-type GStrv): a set of results prepared by the preparation query
510 * @cancellable: a #GCancellable
511 *
512 * The ::writeback-file signal is emitted whenever a file must be written
513 * back
514 *
515 * Returns: %TRUE on success, %FALSE otherwise
516 *
517 * Since: 0.10.20
518 **/
519 signals[WRITEBACK_FILE] =
520 g_signal_new ("writeback-file",
521 G_OBJECT_CLASS_TYPE (object_class),
522 G_SIGNAL_RUN_LAST,
523 G_STRUCT_OFFSET (TrackerMinerFSClass, writeback_file),
524 NULL,
525 NULL,
526 tracker_marshal_BOOLEAN__OBJECT_BOXED_BOXED_OBJECT,
527 G_TYPE_BOOLEAN,
528 4,
529 G_TYPE_FILE,
530 G_TYPE_STRV,
531 G_TYPE_PTR_ARRAY,
532 G_TYPE_CANCELLABLE);
533
534 g_type_class_add_private (object_class, sizeof (TrackerMinerFSPrivate));
535 }
536
537 static void
538 tracker_miner_fs_init (TrackerMinerFS *object)
539 {
540 TrackerMinerFSPrivate *priv;
541
542 object->priv = TRACKER_MINER_FS_GET_PRIVATE (object);
543
544 priv = object->priv;
545
546 priv->timer = g_timer_new ();
547 priv->extraction_timer = g_timer_new ();
548
549 priv->timer_stopped = TRUE;
550 priv->extraction_timer_stopped = TRUE;
551
552 priv->items_created = tracker_priority_queue_new ();
553 priv->items_updated = tracker_priority_queue_new ();
554 priv->items_deleted = tracker_priority_queue_new ();
555 priv->items_moved = tracker_priority_queue_new ();
556 priv->items_writeback = tracker_priority_queue_new ();
557
558 #ifdef EVENT_QUEUE_ENABLE_TRACE
559 priv->queue_status_timeout_id = g_timeout_add_seconds (EVENT_QUEUE_STATUS_TIMEOUT_SECS,
560 miner_fs_queues_status_trace_timeout_cb,
561 object);
562 #endif /* PROCESSING_POOL_ENABLE_TRACE */
563
564 priv->items_ignore_next_update = g_hash_table_new_full (g_str_hash, g_str_equal,
565 (GDestroyNotify) g_free,
566 (GDestroyNotify) NULL);
567
568 /* Create processing pools */
569 priv->task_pool = tracker_task_pool_new (DEFAULT_WAIT_POOL_LIMIT);
570 g_signal_connect (priv->task_pool, "notify::limit-reached",
571 G_CALLBACK (task_pool_limit_reached_notify_cb), object);
572
573 priv->writeback_pool = tracker_task_pool_new (DEFAULT_WAIT_POOL_LIMIT);
574 g_signal_connect (priv->writeback_pool, "notify::limit-reached",
575 G_CALLBACK (task_pool_limit_reached_notify_cb), object);
576
577 /* Create the indexing tree */
578 priv->indexing_tree = tracker_indexing_tree_new ();
579 g_signal_connect (priv->indexing_tree, "directory-removed",
580 G_CALLBACK (indexing_tree_directory_removed),
581 object);
582
583 /* Create the file notifier */
584 priv->file_notifier = tracker_file_notifier_new (priv->indexing_tree);
585
586 g_signal_connect (priv->file_notifier, "file-created",
587 G_CALLBACK (file_notifier_file_created),
588 object);
589 g_signal_connect (priv->file_notifier, "file-updated",
590 G_CALLBACK (file_notifier_file_updated),
591 object);
592 g_signal_connect (priv->file_notifier, "file-deleted",
593 G_CALLBACK (file_notifier_file_deleted),
594 object);
595 g_signal_connect (priv->file_notifier, "file-moved",
596 G_CALLBACK (file_notifier_file_moved),
597 object);
598 g_signal_connect (priv->file_notifier, "directory-started",
599 G_CALLBACK (file_notifier_directory_started),
600 object);
601 g_signal_connect (priv->file_notifier, "directory-finished",
602 G_CALLBACK (file_notifier_directory_finished),
603 object);
604 g_signal_connect (priv->file_notifier, "finished",
605 G_CALLBACK (file_notifier_finished),
606 object);
607
608 priv->quark_ignore_file = g_quark_from_static_string ("tracker-ignore-file");
609 priv->quark_directory_found_crawling = g_quark_from_static_string ("tracker-directory-found-crawling");
610 priv->quark_attribute_updated = g_quark_from_static_string ("tracker-attribute-updated");
611 priv->quark_reentry_counter = g_quark_from_static_string ("tracker-reentry-counter");
612
613 priv->mtime_checking = TRUE;
614 priv->initial_crawling = TRUE;
615 }
616
617 static gboolean
618 miner_fs_initable_init (GInitable *initable,
619 GCancellable *cancellable,
620 GError **error)
621 {
622 TrackerMinerFSPrivate *priv;
623 guint limit;
624
625 if (!miner_fs_initable_parent_iface->init (initable, cancellable, error)) {
626 return FALSE;
627 }
628
629 priv = TRACKER_MINER_FS_GET_PRIVATE (initable);
630
631 g_object_get (initable, "processing-pool-ready-limit", &limit, NULL);
632 priv->sparql_buffer = tracker_sparql_buffer_new (tracker_miner_get_connection (TRACKER_MINER (initable)),
633 limit);
634 g_signal_connect (priv->sparql_buffer, "notify::limit-reached",
635 G_CALLBACK (task_pool_limit_reached_notify_cb),
636 initable);
637
638 return TRUE;
639 }
640
641 static void
642 miner_fs_initable_iface_init (GInitableIface *iface)
643 {
644 miner_fs_initable_parent_iface = g_type_interface_peek_parent (iface);
645 iface->init = miner_fs_initable_init;
646 }
647
648 static void
649 fs_finalize (GObject *object)
650 {
651 TrackerMinerFSPrivate *priv;
652
653 priv = TRACKER_MINER_FS_GET_PRIVATE (object);
654
655 g_timer_destroy (priv->timer);
656 g_timer_destroy (priv->extraction_timer);
657
658 if (priv->item_queues_handler_id) {
659 g_source_remove (priv->item_queues_handler_id);
660 priv->item_queues_handler_id = 0;
661 }
662
663 if (priv->item_queue_blocker) {
664 g_object_unref (priv->item_queue_blocker);
665 }
666
667 tracker_file_notifier_stop (priv->file_notifier);
668
669 /* Cancel every pending task */
670 tracker_task_pool_foreach (priv->task_pool,
671 task_pool_cancel_foreach,
672 NULL);
673 g_object_unref (priv->task_pool);
674
675 g_object_unref (priv->writeback_pool);
676
677 if (priv->sparql_buffer) {
678 g_object_unref (priv->sparql_buffer);
679 }
680
681 tracker_priority_queue_foreach (priv->items_moved,
682 (GFunc) item_moved_data_free,
683 NULL);
684 tracker_priority_queue_unref (priv->items_moved);
685
686 tracker_priority_queue_foreach (priv->items_deleted,
687 (GFunc) g_object_unref,
688 NULL);
689 tracker_priority_queue_unref (priv->items_deleted);
690
691 tracker_priority_queue_foreach (priv->items_updated,
692 (GFunc) g_object_unref,
693 NULL);
694 tracker_priority_queue_unref (priv->items_updated);
695
696 tracker_priority_queue_foreach (priv->items_created,
697 (GFunc) g_object_unref,
698 NULL);
699 tracker_priority_queue_unref (priv->items_created);
700
701 tracker_priority_queue_foreach (priv->items_writeback,
702 (GFunc) item_writeback_data_free,
703 NULL);
704 tracker_priority_queue_unref (priv->items_writeback);
705
706 g_hash_table_unref (priv->items_ignore_next_update);
707
708 g_object_unref (priv->indexing_tree);
709 g_object_unref (priv->file_notifier);
710
711 #ifdef EVENT_QUEUE_ENABLE_TRACE
712 if (priv->queue_status_timeout_id)
713 g_source_remove (priv->queue_status_timeout_id);
714 #endif /* PROCESSING_POOL_ENABLE_TRACE */
715
716 G_OBJECT_CLASS (tracker_miner_fs_parent_class)->finalize (object);
717 }
718
719 static void
720 fs_set_property (GObject *object,
721 guint prop_id,
722 const GValue *value,
723 GParamSpec *pspec)
724 {
725 TrackerMinerFS *fs = TRACKER_MINER_FS (object);
726
727 switch (prop_id) {
728 case PROP_THROTTLE:
729 tracker_miner_fs_set_throttle (TRACKER_MINER_FS (object),
730 g_value_get_double (value));
731 break;
732 case PROP_WAIT_POOL_LIMIT:
733 tracker_task_pool_set_limit (fs->priv->task_pool,
734 g_value_get_uint (value));
735 break;
736 case PROP_READY_POOL_LIMIT:
737 fs->priv->sparql_buffer_limit = g_value_get_uint (value);
738
739 if (fs->priv->sparql_buffer) {
740 tracker_task_pool_set_limit (TRACKER_TASK_POOL (fs->priv->sparql_buffer),
741 fs->priv->sparql_buffer_limit);
742 }
743 break;
744 case PROP_MTIME_CHECKING:
745 fs->priv->mtime_checking = g_value_get_boolean (value);
746 break;
747 case PROP_INITIAL_CRAWLING:
748 fs->priv->initial_crawling = g_value_get_boolean (value);
749 break;
750 default:
751 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
752 break;
753 }
754 }
755
756 static void
757 fs_get_property (GObject *object,
758 guint prop_id,
759 GValue *value,
760 GParamSpec *pspec)
761 {
762 TrackerMinerFS *fs;
763
764 fs = TRACKER_MINER_FS (object);
765
766 switch (prop_id) {
767 case PROP_THROTTLE:
768 g_value_set_double (value, fs->priv->throttle);
769 break;
770 case PROP_WAIT_POOL_LIMIT:
771 g_value_set_uint (value,
772 tracker_task_pool_get_limit (fs->priv->task_pool));
773 break;
774 case PROP_READY_POOL_LIMIT:
775 g_value_set_uint (value, fs->priv->sparql_buffer_limit);
776 break;
777 case PROP_MTIME_CHECKING:
778 g_value_set_boolean (value, fs->priv->mtime_checking);
779 break;
780 case PROP_INITIAL_CRAWLING:
781 g_value_set_boolean (value, fs->priv->initial_crawling);
782 break;
783 default:
784 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
785 break;
786 }
787 }
788
789 static void
790 task_pool_limit_reached_notify_cb (GObject *object,
791 GParamSpec *pspec,
792 gpointer user_data)
793 {
794 if (!tracker_task_pool_limit_reached (TRACKER_TASK_POOL (object))) {
795 item_queue_handlers_set_up (TRACKER_MINER_FS (user_data));
796 }
797 }
798
799 static void
800 miner_started (TrackerMiner *miner)
801 {
802 TrackerMinerFS *fs;
803
804 fs = TRACKER_MINER_FS (miner);
805
806 fs->priv->been_started = TRUE;
807
808 tracker_info ("Initializing");
809
810 g_object_set (miner,
811 "progress", 0.0,
812 "status", "Initializing",
813 "remaining-time", 0,
814 NULL);
815
816 tracker_file_notifier_start (fs->priv->file_notifier);
817 }
818
819 static void
820 miner_stopped (TrackerMiner *miner)
821 {
822 tracker_info ("Idle");
823
824 g_object_set (miner,
825 "progress", 1.0,
826 "status", "Idle",
827 "remaining-time", -1,
828 NULL);
829 }
830
831 static void
832 miner_paused (TrackerMiner *miner)
833 {
834 TrackerMinerFS *fs;
835
836 fs = TRACKER_MINER_FS (miner);
837
838 fs->priv->is_paused = TRUE;
839
840 tracker_file_notifier_stop (fs->priv->file_notifier);
841
842 if (fs->priv->item_queues_handler_id) {
843 g_source_remove (fs->priv->item_queues_handler_id);
844 fs->priv->item_queues_handler_id = 0;
845 }
846 }
847
848 static void
849 miner_resumed (TrackerMiner *miner)
850 {
851 TrackerMinerFS *fs;
852
853 fs = TRACKER_MINER_FS (miner);
854
855 fs->priv->is_paused = FALSE;
856
857 tracker_file_notifier_start (fs->priv->file_notifier);
858
859 /* Only set up queue handler if we have items waiting to be
860 * processed.
861 */
862 if (tracker_miner_fs_has_items_to_process (fs)) {
863 item_queue_handlers_set_up (fs);
864 }
865 }
866
867
868 static void
869 miner_ignore_next_update (TrackerMiner *miner, const GStrv urls)
870 {
871 TrackerMinerFS *fs;
872 guint n;
873
874 fs = TRACKER_MINER_FS (miner);
875
876 for (n = 0; urls[n] != NULL; n++) {
877 g_hash_table_insert (fs->priv->items_ignore_next_update,
878 g_strdup (urls[n]),
879 GINT_TO_POINTER (TRUE));
880 }
881
882 item_queue_handlers_set_up (fs);
883 }
884
885 static void
886 process_print_stats (TrackerMinerFS *fs)
887 {
888 /* Only do this the first time, otherwise the results are
889 * likely to be inaccurate. Devices can be added or removed so
890 * we can't assume stats are correct.
891 */
892 if (!fs->priv->shown_totals) {
893 fs->priv->shown_totals = TRUE;
894
895 tracker_info ("--------------------------------------------------");
896 tracker_info ("Total directories : %d (%d ignored)",
897 fs->priv->total_directories_found,
898 fs->priv->total_directories_ignored);
899 tracker_info ("Total files : %d (%d ignored)",
900 fs->priv->total_files_found,
901 fs->priv->total_files_ignored);
902 #if 0
903 tracker_info ("Total monitors : %d",
904 tracker_monitor_get_count (fs->priv->monitor));
905 #endif
906 tracker_info ("Total processed : %d (%d notified, %d with error)",
907 fs->priv->total_files_processed,
908 fs->priv->total_files_notified,
909 fs->priv->total_files_notified_error);
910 tracker_info ("--------------------------------------------------\n");
911 }
912 }
913
914 static void
915 process_stop (TrackerMinerFS *fs)
916 {
917 /* Now we have finished crawling, print stats and enable monitor events */
918 process_print_stats (fs);
919
920 g_timer_stop (fs->priv->timer);
921 g_timer_stop (fs->priv->extraction_timer);
922
923 fs->priv->timer_stopped = TRUE;
924 fs->priv->extraction_timer_stopped = TRUE;
925
926 tracker_info ("Idle");
927
928 g_object_set (fs,
929 "progress", 1.0,
930 "status", "Idle",
931 "remaining-time", 0,
932 NULL);
933
934 g_signal_emit (fs, signals[FINISHED], 0,
935 g_timer_elapsed (fs->priv->timer, NULL),
936 fs->priv->total_directories_found,
937 fs->priv->total_directories_ignored,
938 fs->priv->total_files_found,
939 fs->priv->total_files_ignored);
940
941 g_timer_stop (fs->priv->timer);
942 g_timer_stop (fs->priv->extraction_timer);
943
944 fs->priv->total_directories_found = 0;
945 fs->priv->total_directories_ignored = 0;
946 fs->priv->total_files_found = 0;
947 fs->priv->total_files_ignored = 0;
948
949 fs->priv->been_crawled = TRUE;
950 }
951
952 static ItemMovedData *
953 item_moved_data_new (GFile *file,
954 GFile *source_file)
955 {
956 ItemMovedData *data;
957
958 data = g_slice_new (ItemMovedData);
959 data->file = g_object_ref (file);
960 data->source_file = g_object_ref (source_file);
961
962 return data;
963 }
964
965 static void
966 item_moved_data_free (ItemMovedData *data)
967 {
968 g_object_unref (data->file);
969 g_object_unref (data->source_file);
970 g_slice_free (ItemMovedData, data);
971 }
972
973 static ItemWritebackData *
974 item_writeback_data_new (GFile *file,
975 GStrv rdf_types,
976 GPtrArray *results)
977 {
978 ItemWritebackData *data;
979
980 data = g_slice_new (ItemWritebackData);
981
982 data->file = g_object_ref (file);
983 data->results = g_ptr_array_ref (results);
984 data->rdf_types = g_strdupv (rdf_types);
985 data->cancellable = g_cancellable_new ();
986 data->notified = FALSE;
987
988 return data;
989 }
990
991 static void
992 item_writeback_data_free (ItemWritebackData *data)
993 {
994 g_object_unref (data->file);
995 g_ptr_array_unref (data->results);
996 g_strfreev (data->rdf_types);
997 g_object_unref (data->cancellable);
998 g_slice_free (ItemWritebackData, data);
999 }
1000
1001 static gboolean
1002 item_queue_is_blocked_by_file (TrackerMinerFS *fs,
1003 GFile *file)
1004 {
1005 g_return_val_if_fail (G_IS_FILE (file), FALSE);
1006
1007 if (fs->priv->item_queue_blocker != NULL &&
1008 (fs->priv->item_queue_blocker == file ||
1009 g_file_equal (fs->priv->item_queue_blocker, file))) {
1010 return TRUE;
1011 }
1012
1013 return FALSE;
1014 }
1015
1016 static void
1017 sparql_buffer_task_finished_cb (GObject *object,
1018 GAsyncResult *result,
1019 gpointer user_data)
1020 {
1021 TrackerMinerFS *fs;
1022 TrackerMinerFSPrivate *priv;
1023 TrackerTask *task;
1024 GFile *task_file;
1025 GError *error = NULL;
1026
1027 fs = user_data;
1028 priv = fs->priv;
1029
1030 if (g_simple_async_result_propagate_error (G_SIMPLE_ASYNC_RESULT (result),
1031 &error)) {
1032 g_critical ("Could not execute sparql: %s", error->message);
1033 priv->total_files_notified_error++;
1034 g_error_free (error);
1035 }
1036
1037 task = g_simple_async_result_get_op_res_gpointer (G_SIMPLE_ASYNC_RESULT (result));
1038 task_file = tracker_task_get_file (task);
1039
1040 if (item_queue_is_blocked_by_file (fs, task_file)) {
1041 g_object_unref (priv->item_queue_blocker);
1042 priv->item_queue_blocker = NULL;
1043 }
1044
1045 if (priv->item_queue_blocker != NULL) {
1046 if (tracker_task_pool_get_size (TRACKER_TASK_POOL (object)) > 0) {
1047 tracker_sparql_buffer_flush (TRACKER_SPARQL_BUFFER (object),
1048 "Item queue still blocked after flush");
1049 }
1050 } else {
1051 item_queue_handlers_set_up (fs);
1052 }
1053 }
1054
1055 static UpdateProcessingTaskContext *
1056 update_processing_task_context_new (TrackerMiner *miner,
1057 gint priority,
1058 const gchar *urn,
1059 const gchar *parent_urn,
1060 GCancellable *cancellable,
1061 TrackerSparqlBuilder *builder)
1062 {
1063 UpdateProcessingTaskContext *ctxt;
1064
1065 ctxt = g_slice_new0 (UpdateProcessingTaskContext);
1066 ctxt->miner = miner;
1067 ctxt->urn = g_strdup (urn);
1068 ctxt->parent_urn = g_strdup (parent_urn);
1069 ctxt->priority = priority;
1070
1071 if (cancellable) {
1072 ctxt->cancellable = g_object_ref (cancellable);
1073 }
1074
1075 if (builder) {
1076 ctxt->builder = g_object_ref (builder);
1077 }
1078
1079 return ctxt;
1080 }
1081
1082 static void
1083 update_processing_task_context_free (UpdateProcessingTaskContext *ctxt)
1084 {
1085 g_free (ctxt->urn);
1086 g_free (ctxt->parent_urn);
1087
1088 if (ctxt->cancellable) {
1089 g_object_unref (ctxt->cancellable);
1090 }
1091
1092 if (ctxt->builder) {
1093 g_object_unref (ctxt->builder);
1094 }
1095
1096 g_slice_free (UpdateProcessingTaskContext, ctxt);
1097 }
1098
1099 static gboolean
1100 do_process_file (TrackerMinerFS *fs,
1101 TrackerTask *task)
1102 {
1103 TrackerMinerFSPrivate *priv;
1104 gboolean processing;
1105 gboolean attribute_update_only;
1106 gchar *uri;
1107 GFile *task_file;
1108 UpdateProcessingTaskContext *ctxt;
1109
1110 ctxt = tracker_task_get_data (task);
1111 task_file = tracker_task_get_file (task);
1112 uri = g_file_get_uri (task_file);
1113 priv = fs->priv;
1114
1115 attribute_update_only = GPOINTER_TO_INT (g_object_get_qdata (G_OBJECT (task_file),
1116 priv->quark_attribute_updated));
1117
1118 if (!attribute_update_only) {
1119 g_debug ("Processing file '%s'...", uri);
1120 g_signal_emit (fs, signals[PROCESS_FILE], 0,
1121 task_file,
1122 ctxt->builder,
1123 ctxt->cancellable,
1124 &processing);
1125 } else {
1126 g_debug ("Processing attributes in file '%s'...", uri);
1127 g_signal_emit (fs, signals[PROCESS_FILE_ATTRIBUTES], 0,
1128 task_file,
1129 ctxt->builder,
1130 ctxt->cancellable,
1131 &processing);
1132 }
1133
1134 if (!processing) {
1135 /* Re-fetch data, since it might have been
1136 * removed in broken implementations
1137 */
1138 task = tracker_task_pool_find (priv->task_pool, task_file);
1139
1140 g_message ("%s refused to process '%s'", G_OBJECT_TYPE_NAME (fs), uri);
1141
1142 if (!task) {
1143 g_critical ("%s has returned FALSE in ::process-file for '%s', "
1144 "but it seems that this file has been processed through "
1145 "tracker_miner_fs_file_notify(), this is an "
1146 "implementation error", G_OBJECT_TYPE_NAME (fs), uri);
1147 } else {
1148 tracker_task_pool_remove (priv->task_pool, task);
1149 tracker_task_unref (task);
1150 }
1151 }
1152
1153 g_free (uri);
1154
1155 return processing;
1156 }
1157
1158 static void
1159 item_add_or_update_cb (TrackerMinerFS *fs,
1160 TrackerTask *extraction_task,
1161 const GError *error)
1162 {
1163 UpdateProcessingTaskContext *ctxt;
1164 TrackerTask *sparql_task = NULL;
1165 GFile *task_file;
1166 gchar *uri;
1167
1168 ctxt = tracker_task_get_data (extraction_task);
1169 task_file = tracker_task_get_file (extraction_task);
1170 uri = g_file_get_uri (task_file);
1171
1172 tracker_task_pool_remove (fs->priv->task_pool, extraction_task);
1173
1174 if (error) {
1175 g_message ("Could not process '%s': %s", uri, error->message);
1176
1177 fs->priv->total_files_notified_error++;
1178
1179 if (!g_error_matches (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND) &&
1180 !g_error_matches (error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) {
1181 sparql_task = tracker_sparql_task_new_with_sparql (task_file,
1182 ctxt->builder);
1183 }
1184 } else {
1185 if (ctxt->urn) {
1186 gboolean attribute_update_only;
1187
1188 attribute_update_only = GPOINTER_TO_INT (g_object_steal_qdata (G_OBJECT (task_file),
1189 fs->priv->quark_attribute_updated));
1190 g_debug ("Updating item '%s' with urn '%s'%s",
1191 uri,
1192 ctxt->urn,
1193 attribute_update_only ? " (attributes only)" : "");
1194
1195 if (!attribute_update_only) {
1196 gchar *full_sparql;
1197
1198 /* Update, delete all statements inserted by miner except:
1199 * - rdf:type statements as they could cause implicit deletion of user data
1200 * - nie:contentCreated so it persists across updates
1201 *
1202 * Additionally, delete also nie:url as it might have been set by 3rd parties,
1203 * and it's used to know whether a file is known to tracker or not.
1204 */
1205 full_sparql = g_strdup_printf ("DELETE {"
1206 " GRAPH <%s> {"
1207 " <%s> ?p ?o"
1208 " } "
1209 "} "
1210 "WHERE {"
1211 " GRAPH <%s> {"
1212 " <%s> ?p ?o"
1213 " FILTER (?p != rdf:type && ?p != nie:contentCreated)"
1214 " } "
1215 "} "
1216 "DELETE {"
1217 " <%s> nie:url ?o"
1218 "} WHERE {"
1219 " <%s> nie:url ?o"
1220 "}"
1221 "%s",
1222 TRACKER_MINER_FS_GRAPH_URN, ctxt->urn,
1223 TRACKER_MINER_FS_GRAPH_URN, ctxt->urn,
1224 ctxt->urn, ctxt->urn,
1225 tracker_sparql_builder_get_result (ctxt->builder));
1226
1227 sparql_task = tracker_sparql_task_new_take_sparql_str (task_file, full_sparql);
1228 } else {
1229 /* Do not drop graph if only updating attributes, the SPARQL builder
1230 * will already contain the necessary DELETE statements for the properties
1231 * being updated */
1232 sparql_task = tracker_sparql_task_new_with_sparql (task_file, ctxt->builder);
1233 }
1234 } else {
1235 g_debug ("Creating new item '%s'", uri);
1236 sparql_task = tracker_sparql_task_new_with_sparql (task_file, ctxt->builder);
1237 }
1238 }
1239
1240 if (sparql_task) {
1241 tracker_sparql_buffer_push (fs->priv->sparql_buffer,
1242 sparql_task,
1243 ctxt->priority,
1244 sparql_buffer_task_finished_cb,
1245 fs);
1246
1247 if (item_queue_is_blocked_by_file (fs, task_file)) {
1248 tracker_sparql_buffer_flush (fs->priv->sparql_buffer, "Current file is blocking item queue");
1249 }
1250 } else {
1251 if (item_queue_is_blocked_by_file (fs, task_file)) {
1252 /* Make sure that we don't stall the item queue, although we could
1253 * expect the file to be reenqueued until the loop detector makes
1254 * us drop it since we were specifically waiting for it to complete.
1255 */
1256 g_object_unref (fs->priv->item_queue_blocker);
1257 fs->priv->item_queue_blocker = NULL;
1258 item_queue_handlers_set_up (fs);
1259 }
1260 }
1261
1262 if (tracker_miner_fs_has_items_to_process (fs) == FALSE &&
1263 tracker_task_pool_get_size (TRACKER_TASK_POOL (fs->priv->task_pool)) == 0) {
1264 /* We need to run this one more time to trigger process_stop() */
1265 item_queue_handlers_set_up (fs);
1266 }
1267
1268 tracker_task_unref (extraction_task);
1269
1270 g_free (uri);
1271 }
1272
1273 static gboolean
1274 item_add_or_update (TrackerMinerFS *fs,
1275 GFile *file,
1276 gint priority,
1277 gboolean is_new)
1278 {
1279 TrackerMinerFSPrivate *priv;
1280 TrackerSparqlBuilder *sparql;
1281 GCancellable *cancellable;
1282 gboolean retval;
1283 TrackerTask *task;
1284 const gchar *parent_urn, *urn = NULL;
1285 UpdateProcessingTaskContext *ctxt;
1286 GFile *parent;
1287
1288 priv = fs->priv;
1289 retval = TRUE;
1290
1291 cancellable = g_cancellable_new ();
1292 sparql = tracker_sparql_builder_new_update ();
1293 g_object_ref (file);
1294
1295 /* Always query. No matter we are notified the file was just
1296 * created, its meta data might already be in the store
1297 * (possibly inserted by other application) - in such a case
1298 * we have to UPDATE, not INSERT. */
1299 urn = tracker_file_notifier_get_file_iri (fs->priv->file_notifier, file);
1300
1301 if (!tracker_indexing_tree_file_is_root (fs->priv->indexing_tree, file)) {
1302 parent = g_file_get_parent (file);
1303 parent_urn = tracker_file_notifier_get_file_iri (fs->priv->file_notifier, parent);
1304 g_object_unref (parent);
1305 } else {
1306 parent_urn = NULL;
1307 }
1308
1309 /* Create task and add it to the pool as a WAIT task (we need to extract
1310 * the file metadata and such) */
1311 ctxt = update_processing_task_context_new (TRACKER_MINER (fs),
1312 priority,
1313 urn,
1314 parent_urn,
1315 cancellable,
1316 sparql);
1317 task = tracker_task_new (file, ctxt,
1318 (GDestroyNotify) update_processing_task_context_free);
1319 tracker_task_pool_add (priv->task_pool, task);
1320
1321 if (do_process_file (fs, task)) {
1322 fs->priv->total_files_processed++;
1323
1324 if (tracker_task_pool_limit_reached (priv->task_pool)) {
1325 retval = FALSE;
1326 }
1327 }
1328
1329 g_object_unref (file);
1330 g_object_unref (cancellable);
1331 g_object_unref (sparql);
1332
1333 return retval;
1334 }
1335
1336 static gboolean
1337 item_remove (TrackerMinerFS *fs,
1338 GFile *file,
1339 gboolean only_children)
1340 {
1341 gchar *uri;
1342 TrackerTask *task;
1343 guint flags = 0;
1344
1345 uri = g_file_get_uri (file);
1346
1347 g_debug ("Removing item: '%s' (Deleted from filesystem or no longer monitored)",
1348 uri);
1349
1350 if (!only_children) {
1351 flags = TRACKER_BULK_MATCH_EQUALS;
1352 } else {
1353 tracker_thumbnailer_remove_add (uri, NULL);
1354 tracker_media_art_queue_remove (uri, NULL);
1355 }
1356
1357 /* FIRST:
1358 * Remove tracker:available for the resources we're going to remove.
1359 * This is done so that unavailability of the resources is marked as soon
1360 * as possible, as the actual delete may take reaaaally a long time
1361 * (removing resources for 30GB of files takes even 30minutes in a 1-CPU
1362 * device). */
1363
1364 /* Add new task to processing pool */
1365 task = tracker_sparql_task_new_bulk (file,
1366 "DELETE { "
1367 " ?f tracker:available true "
1368 "}",
1369 flags | TRACKER_BULK_MATCH_CHILDREN);
1370
1371 tracker_sparql_buffer_push (fs->priv->sparql_buffer,
1372 task,
1373 G_PRIORITY_DEFAULT,
1374 sparql_buffer_task_finished_cb,
1375 fs);
1376
1377 /* SECOND:
1378 * Actually remove all resources. This operation is the one which may take
1379 * a long time.
1380 */
1381
1382 /* Add new task to processing pool */
1383 task = tracker_sparql_task_new_bulk (file,
1384 "DELETE { "
1385 " ?f a rdfs:Resource . "
1386 " ?ie a rdfs:Resource "
1387 "}",
1388 flags |
1389 TRACKER_BULK_MATCH_CHILDREN |
1390 TRACKER_BULK_MATCH_LOGICAL_RESOURCES);
1391
1392 tracker_sparql_buffer_push (fs->priv->sparql_buffer,
1393 task,
1394 G_PRIORITY_DEFAULT,
1395 sparql_buffer_task_finished_cb,
1396 fs);
1397
1398 if (!tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
1399 item_queue_handlers_set_up (fs);
1400 }
1401
1402 g_free (uri);
1403
1404 return TRUE;
1405 }
1406
1407 static gboolean
1408 item_ignore_next_update (TrackerMinerFS *fs,
1409 GFile *file,
1410 GFile *source_file)
1411 {
1412 TrackerSparqlBuilder *sparql;
1413 gchar *uri;
1414 gboolean success = FALSE;
1415 GCancellable *cancellable;
1416 GFile *working_file;
1417
1418 /* While we are in ignore-on-next-update:
1419 * o. We always ignore deletes because it's never the final operation
1420 * of a write. We have a delete when both are null.
1421 * o. A create means the write used rename(). This is the final
1422 * operation of a write and thus we make the update query.
1423 * We have a create when file == null and source_file != null
1424 * o. A move means the write used rename(). This is the final
1425 * operation of a write and thus we make the update query.
1426 * We have a move when both file and source_file aren't null.
1427 * o. A update means the write didn't use rename(). This is the
1428 * final operation of a write and thus we make the update query.
1429 * An update means that file != null and source_file == null. */
1430
1431 /* Happens on delete while in write */
1432 if (!file && !source_file) {
1433 return TRUE;
1434 }
1435
1436 /* Create or update, we are the final one so we make the update query */
1437
1438 if (!file && source_file) {
1439 /* Happens on create while in write */
1440 working_file = source_file;
1441 } else {
1442 /* Happens on update while in write */
1443 working_file = file;
1444 }
1445
1446 uri = g_file_get_uri (working_file);
1447
1448 g_debug ("Updating item: '%s' (IgnoreNextUpdate event)", uri);
1449
1450 cancellable = g_cancellable_new ();
1451 sparql = tracker_sparql_builder_new_update ();
1452 g_object_ref (working_file);
1453
1454 /* IgnoreNextUpdate */
1455 g_signal_emit (fs, signals[IGNORE_NEXT_UPDATE_FILE], 0,
1456 working_file, sparql, cancellable, &success);
1457
1458 if (success) {
1459 gchar *query;
1460
1461 /* Perhaps we should move the DELETE to tracker-miner-files.c?
1462 * Or we add support for DELETE to TrackerSparqlBuilder ofcrs */
1463
1464 query = g_strdup_printf ("DELETE { GRAPH <%s> { "
1465 " ?u nfo:fileSize ?unknown1 ; "
1466 " nfo:fileLastModified ?unknown2 ; "
1467 " nfo:fileLastAccessed ?unknown3 ; "
1468 " nie:mimeType ?unknown4 } "
1469 "} WHERE { GRAPH <%s> { "
1470 " ?u nfo:fileSize ?unknown1 ; "
1471 " nfo:fileLastModified ?unknown2 ; "
1472 " nfo:fileLastAccessed ?unknown3 ; "
1473 " nie:mimeType ?unknown4 ; "
1474 " nie:url \"%s\" } "
1475 "} %s", TRACKER_MINER_FS_GRAPH_URN,
1476 TRACKER_MINER_FS_GRAPH_URN, uri,
1477 tracker_sparql_builder_get_result (sparql));
1478
1479 tracker_sparql_connection_update_async (tracker_miner_get_connection (TRACKER_MINER (fs)),
1480 query,
1481 G_PRIORITY_DEFAULT,
1482 NULL,
1483 NULL,
1484 NULL);
1485
1486 g_free (query);
1487 }
1488
1489 g_hash_table_remove (fs->priv->items_ignore_next_update, uri);
1490
1491 g_object_unref (sparql);
1492 g_object_unref (working_file);
1493 g_object_unref (cancellable);
1494
1495 g_free (uri);
1496
1497 return FALSE;
1498 }
1499
1500 static void
1501 item_update_children_uri_cb (GObject *object,
1502 GAsyncResult *result,
1503 gpointer user_data)
1504 {
1505 RecursiveMoveData *data = user_data;
1506 GError *error = NULL;
1507
1508 TrackerSparqlCursor *cursor = tracker_sparql_connection_query_finish (TRACKER_SPARQL_CONNECTION (object), result, &error);
1509
1510 if (error) {
1511 g_critical ("Could not query children: %s", error->message);
1512 g_error_free (error);
1513 if (cursor) {
1514 g_object_unref (cursor);
1515 }
1516 } else {
1517 while (tracker_sparql_cursor_next (cursor, NULL, NULL)) {
1518 const gchar *child_source_uri, *child_mime, *child_urn;
1519 gchar *child_uri;
1520
1521 child_urn = tracker_sparql_cursor_get_string (cursor, 0, NULL);
1522 child_source_uri = tracker_sparql_cursor_get_string (cursor, 1, NULL);
1523 child_mime = tracker_sparql_cursor_get_string (cursor, 2, NULL);
1524
1525 if (!g_str_has_prefix (child_source_uri, data->source_uri)) {
1526 g_warning ("Child URI '%s' does not start with parent URI '%s'",
1527 child_source_uri,
1528 data->source_uri);
1529 continue;
1530 }
1531
1532 child_uri = g_strdup_printf ("%s%s", data->uri, child_source_uri + strlen (data->source_uri));
1533
1534 g_string_append_printf (data->sparql,
1535 "DELETE { "
1536 " <%s> nie:url ?u "
1537 "} WHERE { "
1538 " <%s> nie:url ?u "
1539 "} ",
1540 child_urn, child_urn);
1541
1542 g_string_append_printf (data->sparql,
1543 "INSERT INTO <%s> {"
1544 " <%s> nie:url \"%s\" "
1545 "} ",
1546 child_urn, child_urn, child_uri);
1547
1548 tracker_thumbnailer_move_add (child_source_uri, child_mime, child_uri);
1549
1550 g_free (child_uri);
1551 }
1552 }
1553
1554 g_object_unref (cursor);
1555
1556 g_main_loop_quit (data->main_loop);
1557 }
1558
1559 static void
1560 item_update_children_uri (TrackerMinerFS *fs,
1561 RecursiveMoveData *move_data,
1562 const gchar *source_uri,
1563 const gchar *uri)
1564 {
1565 gchar *sparql;
1566
1567 sparql = g_strdup_printf ("SELECT ?child ?url nie:mimeType(?child) WHERE { "
1568 " ?child nie:url ?url . "
1569 " FILTER (tracker:uri-is-descendant (\"%s\", ?url)) "
1570 "}",
1571 source_uri);
1572
1573 tracker_sparql_connection_query_async (tracker_miner_get_connection (TRACKER_MINER (fs)),
1574 sparql,
1575 NULL,
1576 item_update_children_uri_cb,
1577 move_data);
1578
1579 g_free (sparql);
1580 }
1581
1582 static gboolean
1583 item_move (TrackerMinerFS *fs,
1584 GFile *file,
1585 GFile *source_file)
1586 {
1587 gchar *uri, *source_uri;
1588 GFileInfo *file_info;
1589 GString *sparql;
1590 RecursiveMoveData move_data;
1591 TrackerTask *task;
1592 const gchar *source_iri;
1593 gchar *display_name;
1594 gboolean source_exists;
1595 GFile *new_parent;
1596 const gchar *new_parent_iri;
1597 TrackerDirectoryFlags source_flags, flags;
1598
1599 uri = g_file_get_uri (file);
1600 source_uri = g_file_get_uri (source_file);
1601
1602 file_info = g_file_query_info (file,
1603 G_FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME ","
1604 G_FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE ","
1605 G_FILE_ATTRIBUTE_STANDARD_TYPE,
1606 G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
1607 NULL, NULL);
1608
1609 /* Get 'source' ID */
1610 source_iri = tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
1611 source_file);
1612 source_exists = (source_iri != NULL);
1613
1614 if (!file_info) {
1615 gboolean retval;
1616
1617 if (source_exists) {
1618 /* Destination file has gone away, ignore dest file and remove source if any */
1619 retval = item_remove (fs, source_file, FALSE);
1620 } else {
1621 /* Destination file went away, and source wasn't indexed either */
1622 retval = TRUE;
1623 }
1624
1625 g_free (source_uri);
1626 g_free (uri);
1627
1628 return retval;
1629 }
1630
1631 g_debug ("Moving item from '%s' to '%s'",
1632 source_uri,
1633 uri);
1634
1635 tracker_thumbnailer_move_add (source_uri,
1636 g_file_info_get_content_type (file_info),
1637 uri);
1638
1639 sparql = g_string_new ("");
1640
1641 /* Delete destination item from store if any */
1642 g_string_append_printf (sparql,
1643 "DELETE { "
1644 " ?urn a rdfs:Resource "
1645 "} WHERE {"
1646 " ?urn nie:url \"%s\" "
1647 "}",
1648 uri);
1649
1650 g_string_append_printf (sparql,
1651 "DELETE { "
1652 " <%s> nfo:fileName ?f ; "
1653 " nie:url ?u ; "
1654 " nie:isStoredAs ?s ; "
1655 " nfo:belongsToContainer ?b"
1656 "} WHERE { "
1657 " <%s> nfo:fileName ?f ; "
1658 " nie:url ?u ; "
1659 " nie:isStoredAs ?s ; "
1660 " nfo:belongsToContainer ?b"
1661 "} ",
1662 source_iri, source_iri);
1663
1664 display_name = tracker_sparql_escape_string (g_file_info_get_display_name (file_info));
1665
1666 /* Get new parent information */
1667 new_parent = g_file_get_parent (file);
1668 new_parent_iri = tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
1669 new_parent);
1670 if (new_parent && new_parent_iri) {
1671 g_string_append_printf (sparql,
1672 "INSERT INTO <%s> {"
1673 " <%s> nfo:fileName \"%s\" ; "
1674 " nie:url \"%s\" ; "
1675 " nie:isStoredAs <%s> ; "
1676 " nfo:belongsToContainer \"%s\""
1677 "}" ,
1678 source_iri, source_iri,
1679 display_name, uri,
1680 source_iri,
1681 new_parent_iri);
1682 } else {
1683 g_warning ("Adding moved item '%s' without nfo:belongsToContainer (new_parent: %p)",
1684 uri, new_parent);
1685 g_string_append_printf (sparql,
1686 "INSERT INTO <%s> {"
1687 " <%s> nfo:fileName \"%s\" ; "
1688 " nie:url \"%s\" ; "
1689 " nie:isStoredAs <%s>"
1690 "} ",
1691 source_iri, source_iri,
1692 display_name, uri,
1693 source_iri);
1694 }
1695
1696 if (new_parent)
1697 g_object_unref (new_parent);
1698 g_free (display_name);
1699
1700 tracker_indexing_tree_get_root (fs->priv->indexing_tree, source_file, &source_flags);
1701
1702 if ((source_flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0 &&
1703 g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) {
1704 tracker_indexing_tree_get_root (fs->priv->indexing_tree,
1705 file, &flags);
1706
1707 if ((flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0) {
1708 /* Update children uris */
1709 move_data.main_loop = g_main_loop_new (NULL, FALSE);
1710 move_data.sparql = sparql;
1711 move_data.source_uri = source_uri;
1712 move_data.uri = uri;
1713
1714 item_update_children_uri (fs, &move_data, source_uri, uri);
1715
1716 g_main_loop_run (move_data.main_loop);
1717
1718 g_main_loop_unref (move_data.main_loop);
1719 } else {
1720 /* A directory is being moved from a recursive location to
1721 * a non-recursive one, mark all children as deleted.
1722 */
1723 item_remove (fs, source_file, TRUE);
1724 }
1725 }
1726
1727 /* Add new task to processing pool */
1728 task = tracker_sparql_task_new_take_sparql_str (file,
1729 g_string_free (sparql,
1730 FALSE));
1731 tracker_sparql_buffer_push (fs->priv->sparql_buffer,
1732 task,
1733 G_PRIORITY_DEFAULT,
1734 sparql_buffer_task_finished_cb,
1735 fs);
1736
1737 if (!tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
1738 item_queue_handlers_set_up (fs);
1739 }
1740
1741 g_free (uri);
1742 g_free (source_uri);
1743 g_object_unref (file_info);
1744
1745 return TRUE;
1746 }
1747
1748 static gboolean
1749 check_ignore_next_update (TrackerMinerFS *fs, GFile *queue_file)
1750 {
1751 gchar *uri = g_file_get_uri (queue_file);
1752 if (g_hash_table_lookup (fs->priv->items_ignore_next_update, uri)) {
1753 g_free (uri);
1754 return TRUE;
1755 }
1756 g_free (uri);
1757 return FALSE;
1758 }
1759
1760 static gboolean
1761 should_wait (TrackerMinerFS *fs,
1762 GFile *file)
1763 {
1764 GFile *parent;
1765
1766 /* Is the item already being processed? */
1767 if (tracker_task_pool_find (fs->priv->task_pool, file) ||
1768 tracker_task_pool_find (fs->priv->writeback_pool, file) ||
1769 tracker_task_pool_find (TRACKER_TASK_POOL (fs->priv->sparql_buffer), file)) {
1770 /* Yes, a previous event on same item currently
1771 * being processed */
1772 fs->priv->item_queue_blocker = g_object_ref (file);
1773 return TRUE;
1774 }
1775
1776 /* Is the item's parent being processed right now? */
1777 parent = g_file_get_parent (file);
1778 if (parent) {
1779 if (tracker_task_pool_find (fs->priv->task_pool, parent) ||
1780 tracker_task_pool_find (TRACKER_TASK_POOL (fs->priv->sparql_buffer), parent)) {
1781 /* Yes, a previous event on the parent of this item
1782 * currently being processed */
1783 fs->priv->item_queue_blocker = parent;
1784 return TRUE;
1785 }
1786
1787 g_object_unref (parent);
1788 }
1789 return FALSE;
1790 }
1791
1792 static gboolean
1793 item_reenqueue_full (TrackerMinerFS *fs,
1794 TrackerPriorityQueue *item_queue,
1795 GFile *queue_file,
1796 gpointer queue_data,
1797 gint priority)
1798 {
1799 gint reentry_counter;
1800 gchar *uri;
1801 gboolean should_wait;
1802
1803 reentry_counter = GPOINTER_TO_INT (g_object_get_qdata (G_OBJECT (queue_file),
1804 fs->priv->quark_reentry_counter));
1805
1806 if (reentry_counter < REENTRY_MAX) {
1807 g_object_set_qdata (G_OBJECT (queue_file),
1808 fs->priv->quark_reentry_counter,
1809 GINT_TO_POINTER (reentry_counter + 1));
1810 tracker_priority_queue_add (item_queue, queue_data, priority);
1811
1812 should_wait = TRUE;
1813 } else {
1814 uri = g_file_get_uri (queue_file);
1815 g_warning ("File '%s' has been reenqueued more than %d times. It will not be indexed.", uri, REENTRY_MAX);
1816 g_free (uri);
1817
1818 /* We must be careful not to return QUEUE_WAIT when there's actually
1819 * nothing left to wait for, or the crawling might never complete.
1820 */
1821 if (tracker_miner_fs_has_items_to_process (fs)) {
1822 should_wait = TRUE;
1823 } else {
1824 should_wait = FALSE;
1825 }
1826 }
1827
1828 return should_wait;
1829 }
1830
1831 static gboolean
1832 item_reenqueue (TrackerMinerFS *fs,
1833 TrackerPriorityQueue *item_queue,
1834 GFile *queue_file,
1835 gint priority)
1836 {
1837 return item_reenqueue_full (fs, item_queue, queue_file, queue_file, priority);
1838 }
1839
1840 static QueueState
1841 item_queue_get_next_file (TrackerMinerFS *fs,
1842 GFile **file,
1843 GFile **source_file,
1844 gint *priority_out)
1845 {
1846 ItemMovedData *data;
1847 ItemWritebackData *wdata;
1848 GFile *queue_file;
1849 gint priority;
1850
1851 /* Writeback items first */
1852 wdata = tracker_priority_queue_pop (fs->priv->items_writeback,
1853 &priority);
1854 if (wdata) {
1855 gboolean processing;
1856
1857 *file = g_object_ref (wdata->file);
1858 *source_file = NULL;
1859 *priority_out = priority;
1860
1861 trace_eq_pop_head ("WRITEBACK", wdata->file);
1862
1863 g_signal_emit (fs, signals[WRITEBACK_FILE], 0,
1864 wdata->file,
1865 wdata->rdf_types,
1866 wdata->results,
1867 wdata->cancellable,
1868 &processing);
1869
1870 if (processing) {
1871 TrackerTask *task;
1872
1873 task = tracker_task_new (wdata->file, wdata,
1874 (GDestroyNotify) item_writeback_data_free);
1875 tracker_task_pool_add (fs->priv->writeback_pool, task);
1876
1877 return QUEUE_WRITEBACK;
1878 } else {
1879 item_writeback_data_free (wdata);
1880 }
1881 }
1882
1883 /* Deleted items second */
1884 queue_file = tracker_priority_queue_pop (fs->priv->items_deleted,
1885 &priority);
1886 if (queue_file) {
1887 *source_file = NULL;
1888
1889 trace_eq_pop_head ("DELETED", queue_file);
1890
1891 /* Do not ignore DELETED event even if file is marked as
1892 IgnoreNextUpdate. We should never see DELETED on update
1893 (atomic rename or in-place update) but we may see DELETED
1894 due to actual file deletion right after update. */
1895
1896 /* If the same item OR its first parent is currently being processed,
1897 * we need to wait for this event */
1898 if (should_wait (fs, queue_file)) {
1899 *file = NULL;
1900
1901 trace_eq_push_head ("DELETED", queue_file, "Should wait");
1902
1903 /* Need to postpone event... */
1904 if (item_reenqueue (fs, fs->priv->items_deleted, queue_file, priority - 1)) {
1905 return QUEUE_WAIT;
1906 } else {
1907 return QUEUE_NONE;
1908 }
1909 }
1910
1911 *file = queue_file;
1912 *priority_out = priority;
1913 return QUEUE_DELETED;
1914 }
1915
1916 /* Created items next */
1917 queue_file = tracker_priority_queue_pop (fs->priv->items_created,
1918 &priority);
1919 if (queue_file) {
1920 *source_file = NULL;
1921
1922 trace_eq_pop_head ("CREATED", queue_file);
1923
1924 /* Note:
1925 * We won't be considering an IgnoreNextUpdate request if
1926 * the event being processed is a CREATED event and the
1927 * file was still unknown to tracker.
1928 */
1929 if (check_ignore_next_update (fs, queue_file)) {
1930 gchar *uri;
1931
1932 uri = g_file_get_uri (queue_file);
1933
1934 if (tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
1935 queue_file) != NULL) {
1936 g_debug ("CREATED event ignored on file '%s' as it already existed, "
1937 " processing as IgnoreNextUpdate...",
1938 uri);
1939 g_free (uri);
1940
1941 return QUEUE_IGNORE_NEXT_UPDATE;
1942 } else {
1943 /* Just remove the IgnoreNextUpdate request */
1944 g_debug ("Skipping the IgnoreNextUpdate request on CREATED event for '%s', file is actually new",
1945 uri);
1946 g_hash_table_remove (fs->priv->items_ignore_next_update, uri);
1947 g_free (uri);
1948 }
1949 }
1950
1951 /* If the same item OR its first parent is currently being processed,
1952 * we need to wait for this event */
1953 if (should_wait (fs, queue_file)) {
1954 *file = NULL;
1955
1956 trace_eq_push_head ("CREATED", queue_file, "Should wait");
1957
1958 /* Need to postpone event... */
1959 if (item_reenqueue (fs, fs->priv->items_created, queue_file, priority - 1)) {
1960 return QUEUE_WAIT;
1961 } else {
1962 return QUEUE_NONE;
1963 }
1964 }
1965
1966 *file = queue_file;
1967 *priority_out = priority;
1968 return QUEUE_CREATED;
1969 }
1970
1971 /* Updated items next */
1972 queue_file = tracker_priority_queue_pop (fs->priv->items_updated,
1973 &priority);
1974 if (queue_file) {
1975 *file = queue_file;
1976 *source_file = NULL;
1977
1978 trace_eq_pop_head ("UPDATED", queue_file);
1979
1980 if (check_ignore_next_update (fs, queue_file)) {
1981 gchar *uri;
1982
1983 uri = g_file_get_uri (queue_file);
1984 g_debug ("UPDATED event ignored on file '%s', "
1985 " processing as IgnoreNextUpdate...",
1986 uri);
1987 g_free (uri);
1988
1989 return QUEUE_IGNORE_NEXT_UPDATE;
1990 }
1991
1992 /* If the same item OR its first parent is currently being processed,
1993 * we need to wait for this event */
1994 if (should_wait (fs, queue_file)) {
1995 *file = NULL;
1996
1997 trace_eq_push_head ("UPDATED", queue_file, "Should wait");
1998
1999 /* Need to postpone event... */
2000 if (item_reenqueue (fs, fs->priv->items_updated, queue_file, priority - 1)) {
2001 return QUEUE_WAIT;
2002 } else {
2003 return QUEUE_NONE;
2004 }
2005 }
2006
2007 *priority_out = priority;
2008
2009 return QUEUE_UPDATED;
2010 }
2011
2012 /* Moved items next */
2013 data = tracker_priority_queue_pop (fs->priv->items_moved,
2014 &priority);
2015 if (data) {
2016 trace_eq_pop_head_2 ("MOVED", data->file, data->source_file);
2017
2018 if (check_ignore_next_update (fs, data->file)) {
2019 gchar *uri;
2020 gchar *source_uri;
2021
2022 uri = g_file_get_uri (queue_file);
2023 source_uri = g_file_get_uri (data->source_file);
2024 g_debug ("MOVED event ignored on files '%s->%s', "
2025 " processing as IgnoreNextUpdate on '%s'",
2026 source_uri, uri, uri);
2027 g_free (uri);
2028 g_free (source_uri);
2029
2030 *file = g_object_ref (data->file);
2031 *source_file = g_object_ref (data->source_file);
2032 item_moved_data_free (data);
2033 return QUEUE_IGNORE_NEXT_UPDATE;
2034 }
2035
2036 /* If the same item OR its first parent is currently being processed,
2037 * we need to wait for this event */
2038 if (should_wait (fs, data->file) ||
2039 should_wait (fs, data->source_file)) {
2040 *file = NULL;
2041 *source_file = NULL;
2042
2043 trace_eq_push_head_2 ("MOVED", data->source_file, data->file, "Should wait");
2044
2045 /* Need to postpone event... */
2046 if (item_reenqueue_full (fs, fs->priv->items_moved, data->file, data, priority - 1)) {
2047 return QUEUE_WAIT;
2048 } else {
2049 return QUEUE_NONE;
2050 }
2051 }
2052
2053 *file = g_object_ref (data->file);
2054 *source_file = g_object_ref (data->source_file);
2055 *priority_out = priority;
2056 item_moved_data_free (data);
2057 return QUEUE_MOVED;
2058 }
2059
2060 *file = NULL;
2061 *source_file = NULL;
2062
2063 if (tracker_file_notifier_is_active (fs->priv->file_notifier) ||
2064 tracker_task_pool_limit_reached (fs->priv->task_pool) ||
2065 tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
2066 if (tracker_task_pool_get_size (fs->priv->task_pool) == 0) {
2067 fs->priv->extraction_timer_stopped = TRUE;
2068 g_timer_stop (fs->priv->extraction_timer);
2069 }
2070
2071 /* There are still pending items to crawl,
2072 * or extract pool limit is reached
2073 */
2074 return QUEUE_WAIT;
2075 }
2076
2077 return QUEUE_NONE;
2078 }
2079
2080 static gdouble
2081 item_queue_get_progress (TrackerMinerFS *fs,
2082 guint *n_items_processed,
2083 guint *n_items_remaining)
2084 {
2085 guint items_to_process = 0;
2086 guint items_total = 0;
2087
2088 items_to_process += tracker_priority_queue_get_length (fs->priv->items_deleted);
2089 items_to_process += tracker_priority_queue_get_length (fs->priv->items_created);
2090 items_to_process += tracker_priority_queue_get_length (fs->priv->items_updated);
2091 items_to_process += tracker_priority_queue_get_length (fs->priv->items_moved);
2092 items_to_process += tracker_priority_queue_get_length (fs->priv->items_writeback);
2093
2094 items_total += fs->priv->total_directories_found;
2095 items_total += fs->priv->total_files_found;
2096
2097 if (n_items_processed) {
2098 *n_items_processed = ((items_total >= items_to_process) ?
2099 (items_total - items_to_process) : 0);
2100 }
2101
2102 if (n_items_remaining) {
2103 *n_items_remaining = items_to_process;
2104 }
2105
2106 if (items_total == 0 ||
2107 items_to_process == 0 ||
2108 items_to_process > items_total) {
2109 return 1.0;
2110 }
2111
2112 return (gdouble) (items_total - items_to_process) / items_total;
2113 }
2114
2115 static gboolean
2116 item_queue_handlers_cb (gpointer user_data)
2117 {
2118 TrackerMinerFS *fs = user_data;
2119 GFile *file = NULL;
2120 GFile *source_file = NULL;
2121 GFile *parent;
2122 QueueState queue;
2123 GTimeVal time_now;
2124 static GTimeVal time_last = { 0 };
2125 gboolean keep_processing = TRUE;
2126 gint priority = 0;
2127
2128 if (fs->priv->timer_stopped) {
2129 g_timer_start (fs->priv->timer);
2130 fs->priv->timer_stopped = FALSE;
2131 }
2132
2133 if (tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
2134 /* Task pool is full, give it a break */
2135 fs->priv->item_queues_handler_id = 0;
2136 return FALSE;
2137 }
2138
2139 queue = item_queue_get_next_file (fs, &file, &source_file, &priority);
2140
2141 if (queue == QUEUE_WAIT) {
2142 /* Items are still being processed, so wait until
2143 * the processing pool is cleared before starting with
2144 * the next directories batch.
2145 */
2146 fs->priv->item_queues_handler_id = 0;
2147
2148 /* We should flush the processing pool buffer here, because
2149 * if there was a previous task on the same file we want to
2150 * process now, we want it to get finished before we can go
2151 * on with the queues... */
2152 tracker_sparql_buffer_flush (fs->priv->sparql_buffer,
2153 "Queue handlers WAIT");
2154
2155 return FALSE;
2156 }
2157
2158 if (file && queue != QUEUE_DELETED &&
2159 tracker_file_is_locked (file)) {
2160 gchar *uri;
2161
2162 /* File is locked, ignore any updates on it */
2163
2164 uri = g_file_get_uri (file);
2165 g_debug ("File '%s' is currently locked, ignoring updates on it",
2166 uri);
2167 g_free (uri);
2168
2169 g_object_unref (file);
2170
2171 if (source_file) {
2172 g_object_unref (source_file);
2173 }
2174
2175 return TRUE;
2176 }
2177
2178 if (queue == QUEUE_NONE) {
2179 g_timer_stop (fs->priv->extraction_timer);
2180 fs->priv->extraction_timer_stopped = TRUE;
2181 } else if (fs->priv->extraction_timer_stopped) {
2182 g_timer_continue (fs->priv->extraction_timer);
2183 fs->priv->extraction_timer_stopped = FALSE;
2184 }
2185
2186 /* Update progress, but don't spam it. */
2187 g_get_current_time (&time_now);
2188
2189 if ((time_now.tv_sec - time_last.tv_sec) >= 1) {
2190 guint items_processed, items_remaining;
2191 gdouble progress_now;
2192 static gdouble progress_last = 0.0;
2193 static gint info_last = 0;
2194 gdouble seconds_elapsed, extraction_elapsed;
2195
2196 time_last = time_now;
2197
2198 /* Update progress? */
2199 progress_now = item_queue_get_progress (fs,
2200 &items_processed,
2201 &items_remaining);
2202 seconds_elapsed = g_timer_elapsed (fs->priv->timer, NULL);
2203 extraction_elapsed = g_timer_elapsed (fs->priv->extraction_timer, NULL);
2204
2205 if (!tracker_file_notifier_is_active (fs->priv->file_notifier)) {
2206 gchar *status;
2207 gint remaining_time;
2208
2209 g_object_get (fs, "status", &status, NULL);
2210
2211 /* Compute remaining time */
2212 remaining_time = (gint)tracker_seconds_estimate (extraction_elapsed,
2213 items_processed,
2214 items_remaining);
2215
2216 /* CLAMP progress so it doesn't go back below
2217 * 2% (which we use for crawling)
2218 */
2219 if (g_strcmp0 (status, "Processing…") != 0) {
2220 /* Don't spam this */
2221 tracker_info ("Processing…");
2222 g_object_set (fs,
2223 "status", "Processing…",
2224 "progress", CLAMP (progress_now, 0.02, 1.00),
2225 "remaining-time", remaining_time,
2226 NULL);
2227 } else {
2228 g_object_set (fs,
2229 "progress", CLAMP (progress_now, 0.02, 1.00),
2230 "remaining-time", remaining_time,
2231 NULL);
2232 }
2233
2234 g_free (status);
2235 }
2236
2237 if (++info_last >= 5 &&
2238 (gint) (progress_last * 100) != (gint) (progress_now * 100)) {
2239 gchar *str1, *str2;
2240
2241 info_last = 0;
2242 progress_last = progress_now;
2243
2244 /* Log estimated remaining time */
2245 str1 = tracker_seconds_estimate_to_string (extraction_elapsed,
2246 TRUE,
2247 items_processed,
2248 items_remaining);
2249 str2 = tracker_seconds_to_string (seconds_elapsed, TRUE);
2250
2251 tracker_info ("Processed %u/%u, estimated %s left, %s elapsed",
2252 items_processed,
2253 items_processed + items_remaining,
2254 str1,
2255 str2);
2256
2257 g_free (str2);
2258 g_free (str1);
2259 }
2260 }
2261
2262 /* Handle queues */
2263 switch (queue) {
2264 case QUEUE_NONE:
2265 if (!tracker_file_notifier_is_active (fs->priv->file_notifier) &&
2266 tracker_task_pool_get_size (fs->priv->task_pool) == 0) {
2267 if (tracker_task_pool_get_size (TRACKER_TASK_POOL (fs->priv->sparql_buffer)) == 0) {
2268 /* Print stats and signal finished */
2269 process_stop (fs);
2270
2271 tracker_thumbnailer_send ();
2272 tracker_media_art_queue_empty (tracker_miner_get_connection (TRACKER_MINER (fs)));
2273 } else {
2274 /* Flush any possible pending update here */
2275 tracker_sparql_buffer_flush (fs->priv->sparql_buffer,
2276 "Queue handlers NONE");
2277 }
2278 }
2279
2280 /* No more files left to process */
2281 keep_processing = FALSE;
2282 break;
2283 case QUEUE_MOVED:
2284 keep_processing = item_move (fs, file, source_file);
2285 break;
2286 case QUEUE_DELETED:
2287 keep_processing = item_remove (fs, file, FALSE);
2288 break;
2289 case QUEUE_CREATED:
2290 case QUEUE_UPDATED:
2291 parent = g_file_get_parent (file);
2292
2293 if (!parent ||
2294 tracker_indexing_tree_file_is_root (fs->priv->indexing_tree, file) ||
2295 tracker_file_notifier_get_file_iri (fs->priv->file_notifier, parent)) {
2296 keep_processing = item_add_or_update (fs, file, priority,
2297 (queue == QUEUE_CREATED));
2298 } else {
2299 TrackerPriorityQueue *item_queue;
2300 gchar *uri;
2301
2302 uri = g_file_get_uri (parent);
2303 g_message ("Parent '%s' not indexed yet", uri);
2304 g_free (uri);
2305
2306 if (queue == QUEUE_CREATED) {
2307 item_queue = fs->priv->items_created;
2308 } else {
2309 item_queue = fs->priv->items_updated;
2310 }
2311
2312 /* Parent isn't indexed yet, reinsert the task into the queue,
2313 * but forcily prepended by its parent so its indexing is
2314 * ensured, tasks are inserted at a higher priority so they
2315 * are processed promptly anyway.
2316 */
2317 item_reenqueue (fs, item_queue, g_object_ref (parent), priority - 1);
2318 item_reenqueue (fs, item_queue, g_object_ref (file), priority);
2319
2320 keep_processing = TRUE;
2321 }
2322
2323 if (parent) {
2324 g_object_unref (parent);
2325 }
2326
2327 break;
2328 case QUEUE_IGNORE_NEXT_UPDATE:
2329 keep_processing = item_ignore_next_update (fs, file, source_file);
2330 break;
2331 case QUEUE_WRITEBACK:
2332 /* Nothing to do here */
2333 keep_processing = TRUE;
2334 break;
2335 default:
2336 g_assert_not_reached ();
2337 }
2338
2339 if (file) {
2340 g_object_unref (file);
2341 }
2342
2343 if (source_file) {
2344 g_object_unref (source_file);
2345 }
2346
2347 if (!keep_processing) {
2348 fs->priv->item_queues_handler_id = 0;
2349 return FALSE;
2350 } else {
2351 return TRUE;
2352 }
2353 }
2354
2355 static guint
2356 _tracker_idle_add (TrackerMinerFS *fs,
2357 GSourceFunc func,
2358 gpointer user_data)
2359 {
2360 guint interval;
2361
2362 interval = TRACKER_MAX_TIMEOUT_INTERVAL * fs->priv->throttle;
2363
2364 if (interval == 0) {
2365 return g_idle_add_full (TRACKER_TASK_PRIORITY, func, user_data, NULL);
2366 } else {
2367 return g_timeout_add_full (TRACKER_TASK_PRIORITY, interval, func, user_data, NULL);
2368 }
2369 }
2370
2371 static void
2372 item_queue_handlers_set_up (TrackerMinerFS *fs)
2373 {
2374 trace_eq ("Setting up queue handlers...");
2375 if (fs->priv->item_queues_handler_id != 0) {
2376 trace_eq (" cancelled: already one active");
2377 return;
2378 }
2379
2380 if (fs->priv->is_paused) {
2381 trace_eq (" cancelled: paused");
2382 return;
2383 }
2384
2385 if (fs->priv->item_queue_blocker) {
2386 trace_eq (" cancelled: item queue blocked waiting for file '%s'",
2387 g_file_get_path (fs->priv->item_queue_blocker));
2388 return;
2389 }
2390
2391 /* Already sent max number of tasks to tracker-extract/writeback? */
2392 if (tracker_task_pool_limit_reached (fs->priv->task_pool) ||
2393 tracker_task_pool_limit_reached (fs->priv->writeback_pool)) {
2394 trace_eq (" cancelled: pool limit reached (tasks: %u (max %u) , writeback: %u (max %u))",
2395 tracker_task_pool_get_size (fs->priv->task_pool),
2396 tracker_task_pool_get_limit (fs->priv->task_pool),
2397 tracker_task_pool_get_size (fs->priv->writeback_pool),
2398 tracker_task_pool_get_limit (fs->priv->writeback_pool));
2399 return;
2400 }
2401
2402 if (tracker_task_pool_limit_reached (TRACKER_TASK_POOL (fs->priv->sparql_buffer))) {
2403 trace_eq (" cancelled: pool limit reached (sparql buffer: %u)",
2404 tracker_task_pool_get_limit (TRACKER_TASK_POOL (fs->priv->sparql_buffer)));
2405 return;
2406 }
2407
2408 if (!tracker_file_notifier_is_active (fs->priv->file_notifier)) {
2409 gchar *status;
2410 gdouble progress;
2411
2412 g_object_get (fs,
2413 "progress", &progress,
2414 "status", &status,
2415 NULL);
2416
2417 /* Don't spam this */
2418 if (progress > 0.01 && g_strcmp0 (status, "Processing…") != 0) {
2419 tracker_info ("Processing…");
2420 g_object_set (fs, "status", "Processing…", NULL);
2421 }
2422
2423 g_free (status);
2424 }
2425
2426 trace_eq (" scheduled in idle");
2427 fs->priv->item_queues_handler_id =
2428 _tracker_idle_add (fs,
2429 item_queue_handlers_cb,
2430 fs);
2431 }
2432
2433 static gboolean
2434 should_check_file (TrackerMinerFS *fs,
2435 GFile *file,
2436 gboolean is_dir)
2437 {
2438 GFileType file_type;
2439
2440 file_type = (is_dir) ? G_FILE_TYPE_DIRECTORY : G_FILE_TYPE_REGULAR;
2441 return tracker_indexing_tree_file_is_indexable (fs->priv->indexing_tree,
2442 file, file_type);
2443 }
2444
2445 static gboolean
2446 moved_files_equal (gconstpointer a,
2447 gconstpointer b)
2448 {
2449 const ItemMovedData *data = a;
2450 GFile *file = G_FILE (b);
2451
2452 /* Compare with dest file */
2453 return g_file_equal (data->file, file);
2454 }
2455
2456 static gboolean
2457 writeback_files_equal (gconstpointer a,
2458 gconstpointer b)
2459 {
2460 const ItemWritebackData *data = a;
2461 GFile *file = G_FILE (b);
2462
2463 /* Compare with dest file */
2464 return g_file_equal (data->file, file);
2465 }
2466
2467 static gboolean
2468 remove_writeback_task (TrackerMinerFS *fs,
2469 GFile *file)
2470 {
2471 TrackerTask *task;
2472 ItemWritebackData *data;
2473
2474 task = tracker_task_pool_find (fs->priv->writeback_pool, file);
2475
2476 if (!task) {
2477 return FALSE;
2478 }
2479
2480 data = tracker_task_get_data (task);
2481
2482 if (data->notified) {
2483 tracker_task_pool_remove (fs->priv->writeback_pool, task);
2484 tracker_task_unref (task);
2485 return TRUE;
2486 }
2487
2488 return FALSE;
2489 }
2490
2491 static void
2492 cancel_writeback_task (TrackerMinerFS *fs,
2493 GFile *file)
2494 {
2495 TrackerTask *task;
2496
2497 task = tracker_task_pool_find (fs->priv->writeback_pool, file);
2498
2499 if (task) {
2500 ItemWritebackData *data;
2501
2502 data = tracker_task_get_data (task);
2503 g_cancellable_cancel (data->cancellable);
2504 tracker_task_pool_remove (fs->priv->writeback_pool, task);
2505 tracker_task_unref (task);
2506 }
2507 }
2508
2509 /* Checks previous created/updated/deleted/moved/writeback queues for
2510 * monitor events. Returns TRUE if the item should still
2511 * be added to the queue.
2512 */
2513 static gboolean
2514 check_item_queues (TrackerMinerFS *fs,
2515 QueueState queue,
2516 GFile *file,
2517 GFile *other_file)
2518 {
2519 ItemMovedData *move_data;
2520
2521 if (!fs->priv->been_crawled) {
2522 /* Only do this after initial crawling, so
2523 * we are mostly sure that we won't be doing
2524 * checks on huge lists.
2525 */
2526 return TRUE;
2527 }
2528
2529 if (queue == QUEUE_UPDATED) {
2530 TrackerTask *task;
2531
2532 if (other_file) {
2533 task = tracker_task_pool_find (fs->priv->writeback_pool, other_file);
2534 } else {
2535 task = tracker_task_pool_find (fs->priv->writeback_pool, file);
2536 }
2537
2538 if (task) {
2539 /* There is a writeback task for
2540 * this file, so avoid any updates
2541 */
2542 return FALSE;
2543 }
2544 }
2545
2546 switch (queue) {
2547 case QUEUE_CREATED:
2548 /* Created items aren't likely to have
2549 * anything in other queues for the same
2550 * file.
2551 */
2552 return TRUE;
2553 case QUEUE_UPDATED:
2554 /* No further updates after a previous created/updated event */
2555 if (tracker_priority_queue_find (fs->priv->items_created, NULL,
2556 (GEqualFunc) g_file_equal, file) ||
2557 tracker_priority_queue_find (fs->priv->items_updated, NULL,
2558 (GEqualFunc) g_file_equal, file)) {
2559 g_debug (" Found previous unhandled CREATED/UPDATED event");
2560 return FALSE;
2561 }
2562 case QUEUE_WRITEBACK:
2563 /* No consecutive writebacks for the same file */
2564 if (tracker_priority_queue_find (fs->priv->items_writeback, NULL,
2565 writeback_files_equal, file)) {
2566 g_debug (" Found previous unhandled WRITEBACK event");
2567 return FALSE;
2568 }
2569
2570 return TRUE;
2571 case QUEUE_DELETED:
2572 if (tracker_task_pool_find (fs->priv->writeback_pool, file)) {
2573 /* Cancel writeback operations on a deleted file */
2574 cancel_writeback_task (fs, file);
2575 }
2576
2577 /* Remove all previous updates */
2578 if (tracker_priority_queue_foreach_remove (fs->priv->items_updated,
2579 (GEqualFunc) g_file_equal,
2580 file,
2581 (GDestroyNotify) g_object_unref)) {
2582 g_debug (" Deleting previous unhandled UPDATED event");
2583 }
2584
2585 if (tracker_priority_queue_foreach_remove (fs->priv->items_created,
2586 (GEqualFunc) g_file_equal,
2587 file,
2588 (GDestroyNotify) g_object_unref)) {
2589 /* Created event was still in the queue,
2590 * remove it and ignore the current event
2591 */
2592 g_debug (" Found matching unhandled CREATED event, removing file altogether");
2593 return FALSE;
2594 }
2595
2596 return TRUE;
2597 case QUEUE_MOVED:
2598 if (tracker_task_pool_find (fs->priv->writeback_pool, file)) {
2599 /* If the origin file is also being written back,
2600 * cancel it as this is an external operation.
2601 */
2602 cancel_writeback_task (fs, file);
2603 }
2604
2605 /* Kill any events on other_file (The dest one), since it will be rewritten anyway */
2606 if (tracker_priority_queue_foreach_remove (fs->priv->items_created,
2607 (GEqualFunc) g_file_equal,
2608 other_file,
2609 (GDestroyNotify) g_object_unref)) {
2610 g_debug (" Removing previous unhandled CREATED event for dest file, will be rewritten anyway");
2611 }
2612
2613 if (tracker_priority_queue_foreach_remove (fs->priv->items_updated,
2614 (GEqualFunc) g_file_equal,
2615 other_file,
2616 (GDestroyNotify) g_object_unref)) {
2617 g_debug (" Removing previous unhandled UPDATED event for dest file, will be rewritten anyway");
2618 }
2619
2620 /* Now check file (Origin one) */
2621 if (tracker_priority_queue_foreach_remove (fs->priv->items_created,
2622 (GEqualFunc) g_file_equal,
2623 file,
2624 (GDestroyNotify) g_object_unref)) {
2625 /* If source file was created, replace it with
2626 * a create event for the destination file, and
2627 * discard this event.
2628 *
2629 * We assume all posterior updates
2630 * have been merged together previously by this
2631 * same function.
2632 */
2633 g_debug (" Found matching unhandled CREATED event "
2634 "for source file, merging both events together");
2635 tracker_priority_queue_add (fs->priv->items_created,
2636 g_object_ref (other_file),
2637 G_PRIORITY_DEFAULT);
2638
2639 return FALSE;
2640 }
2641
2642 move_data = tracker_priority_queue_find (fs->priv->items_moved, NULL,
2643 (GEqualFunc) moved_files_equal, file);
2644 if (move_data) {
2645 /* Origin file was the dest of a previous
2646 * move operation, merge these together.
2647 */
2648 g_debug (" Source file is the destination of a previous "
2649 "unhandled MOVED event, merging both events together");
2650 g_object_unref (move_data->file);
2651 move_data->file = g_object_ref (other_file);
2652 return FALSE;
2653 }
2654
2655 return TRUE;
2656 break;
2657 default:
2658 g_assert_not_reached ();
2659 }
2660
2661 return TRUE;
2662 }
2663
2664 static void
2665 file_notifier_file_created (TrackerFileNotifier *notifier,
2666 GFile *file,
2667 gpointer user_data)
2668 {
2669 TrackerMinerFS *fs = user_data;
2670
2671 if (check_item_queues (fs, QUEUE_CREATED, file, NULL)) {
2672 tracker_priority_queue_add (fs->priv->items_created,
2673 g_object_ref (file),
2674 G_PRIORITY_DEFAULT);
2675 item_queue_handlers_set_up (fs);
2676 }
2677 }
2678
2679 static void
2680 file_notifier_file_deleted (TrackerFileNotifier *notifier,
2681 GFile *file,
2682 gpointer user_data)
2683 {
2684 TrackerMinerFS *fs = user_data;
2685
2686 if (check_item_queues (fs, QUEUE_DELETED, file, NULL)) {
2687 tracker_priority_queue_add (fs->priv->items_deleted,
2688 g_object_ref (file),
2689 G_PRIORITY_DEFAULT);
2690 item_queue_handlers_set_up (fs);
2691 }
2692 }
2693
2694 static void
2695 file_notifier_file_updated (TrackerFileNotifier *notifier,
2696 GFile *file,
2697 gboolean attributes_only,
2698 gpointer user_data)
2699 {
2700 TrackerMinerFS *fs = user_data;
2701
2702 /* Writeback tasks would receive an updated after move,
2703 * consequence of the data being written back in the
2704 * copy, and its monitor events being propagated to
2705 * the destination file.
2706 */
2707 if (!attributes_only &&
2708 remove_writeback_task (fs, file)) {
2709 item_queue_handlers_set_up (fs);
2710 return;
2711 }
2712
2713 if (check_item_queues (fs, QUEUE_UPDATED, file, NULL)) {
2714 if (attributes_only) {
2715 g_object_set_qdata (G_OBJECT (file),
2716 fs->priv->quark_attribute_updated,
2717 GINT_TO_POINTER (TRUE));
2718 }
2719
2720 tracker_priority_queue_add (fs->priv->items_updated,
2721 g_object_ref (file),
2722 G_PRIORITY_DEFAULT);
2723 item_queue_handlers_set_up (fs);
2724 }
2725 }
2726
2727 static void
2728 file_notifier_file_moved (TrackerFileNotifier *notifier,
2729 GFile *source,
2730 GFile *dest,
2731 gpointer user_data)
2732 {
2733 TrackerMinerFS *fs = user_data;
2734
2735 if (check_item_queues (fs, QUEUE_MOVED, source, dest)) {
2736 tracker_priority_queue_add (fs->priv->items_moved,
2737 item_moved_data_new (dest, source),
2738 G_PRIORITY_DEFAULT);
2739 item_queue_handlers_set_up (fs);
2740 }
2741 }
2742
2743 static void
2744 file_notifier_directory_started (TrackerFileNotifier *notifier,
2745 GFile *directory,
2746 gpointer user_data)
2747 {
2748 TrackerMinerFS *fs = user_data;
2749 TrackerDirectoryFlags flags;
2750 gchar *str, *uri;
2751
2752 uri = g_file_get_uri (directory);
2753 tracker_indexing_tree_get_root (fs->priv->indexing_tree,
2754 directory, &flags);
2755
2756 if ((flags & TRACKER_DIRECTORY_FLAG_RECURSE) != 0) {
2757 str = g_strdup_printf ("Crawling recursively directory '%s'", uri);
2758 } else {
2759 str = g_strdup_printf ("Crawling single directory '%s'", uri);
2760 }
2761
2762 if (fs->priv->timer_stopped) {
2763 g_timer_start (fs->priv->timer);
2764 fs->priv->timer_stopped = FALSE;
2765 }
2766
2767 if (fs->priv->extraction_timer_stopped) {
2768 g_timer_start (fs->priv->timer);
2769 fs->priv->extraction_timer_stopped = FALSE;
2770 }
2771
2772 /* Always set the progress here to at least 1%, and the remaining time
2773 * to -1 as we cannot guess during crawling (we don't know how many directories
2774 * we will find) */
2775 g_object_set (fs,
2776 "progress", 0.01,
2777 "status", str,
2778 "remaining-time", -1,
2779 NULL);
2780 g_free (str);
2781 g_free (uri);
2782 }
2783
2784 static void
2785 file_notifier_directory_finished (TrackerFileNotifier *notifier,
2786 GFile *directory,
2787 guint directories_found,
2788 guint directories_ignored,
2789 guint files_found,
2790 guint files_ignored,
2791 gpointer user_data)
2792 {
2793 TrackerMinerFS *fs = user_data;
2794
2795 /* Update stats */
2796 fs->priv->directories_found += directories_found;
2797 fs->priv->directories_ignored += directories_ignored;
2798 fs->priv->files_found += files_found;
2799 fs->priv->files_ignored += files_ignored;
2800
2801 fs->priv->total_directories_found += directories_found;
2802 fs->priv->total_directories_ignored += directories_ignored;
2803 fs->priv->total_files_found += files_found;
2804 fs->priv->total_files_ignored += files_ignored;
2805 }
2806
2807 static void
2808 file_notifier_finished (TrackerFileNotifier *notifier,
2809 gpointer user_data)
2810 {
2811 TrackerMinerFS *fs = user_data;
2812
2813 if (!tracker_miner_fs_has_items_to_process (fs)) {
2814 tracker_info ("Finished all tasks");
2815 process_stop (fs);
2816 }
2817 }
2818
2819
2820 #ifdef CRAWLED_TREE_ENABLE_TRACE
2821
2822 static gboolean
2823 print_file_tree (GNode *node,
2824 gpointer user_data)
2825 {
2826 gchar *name;
2827 gint i;
2828
2829 name = g_file_get_basename (node->data);
2830
2831 /* Indentation */
2832 for (i = g_node_depth (node) - 1; i > 0; i--) {
2833 g_print (" ");
2834 }
2835
2836 g_print ("%s\n", name);
2837 g_free (name);
2838
2839 return FALSE;
2840 }
2841
2842 #endif /* CRAWLED_TREE_ENABLE_TRACE */
2843
2844 /* Returns TRUE if file equals to
2845 * other_file, or is a child of it
2846 */
2847 static gboolean
2848 file_equal_or_descendant (GFile *file,
2849 GFile *prefix)
2850 {
2851 if (g_file_equal (file, prefix) ||
2852 g_file_has_prefix (file, prefix)) {
2853 return TRUE;
2854 }
2855
2856 return FALSE;
2857 }
2858
2859 /**
2860 * tracker_miner_fs_directory_add:
2861 * @fs: a #TrackerMinerFS
2862 * @file: #GFile for the directory to inspect
2863 * @recurse: whether the directory should be inspected recursively
2864 *
2865 * Tells the filesystem miner to inspect a directory.
2866 *
2867 * Since: 0.8
2868 **/
2869 void
2870 tracker_miner_fs_directory_add (TrackerMinerFS *fs,
2871 GFile *file,
2872 gboolean recurse)
2873 {
2874 TrackerDirectoryFlags flags;
2875
2876 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
2877 g_return_if_fail (G_IS_FILE (file));
2878
2879 flags = TRACKER_DIRECTORY_FLAG_MONITOR;
2880
2881 if (recurse) {
2882 flags |= TRACKER_DIRECTORY_FLAG_RECURSE;
2883 }
2884
2885 if (fs->priv->mtime_checking) {
2886 flags |= TRACKER_DIRECTORY_FLAG_CHECK_MTIME;
2887 }
2888
2889 tracker_indexing_tree_add (fs->priv->indexing_tree,
2890 file, flags);
2891 }
2892
2893 static void
2894 task_pool_cancel_foreach (gpointer data,
2895 gpointer user_data)
2896 {
2897 TrackerTask *task = data;
2898 GFile *file = user_data;
2899 GFile *task_file;
2900 UpdateProcessingTaskContext *ctxt;
2901
2902 ctxt = tracker_task_get_data (task);
2903 task_file = tracker_task_get_file (task);
2904
2905 if (ctxt &&
2906 ctxt->cancellable &&
2907 (!file ||
2908 (g_file_equal (task_file, file) ||
2909 g_file_has_prefix (task_file, file)))) {
2910 g_cancellable_cancel (ctxt->cancellable);
2911 }
2912 }
2913
2914 static void
2915 writeback_pool_cancel_foreach (gpointer data,
2916 gpointer user_data)
2917 {
2918 GFile *task_file, *file;
2919 TrackerTask *task;
2920
2921 task = data;
2922 file = user_data;
2923 task_file = tracker_task_get_file (task);
2924
2925 if (!file ||
2926 g_file_equal (task_file, file) ||
2927 g_file_has_prefix (task_file, file)) {
2928 ItemWritebackData *task_data;
2929
2930 task_data = tracker_task_get_data (task);
2931 g_cancellable_cancel (task_data->cancellable);
2932 }
2933 }
2934
2935 static void
2936 indexing_tree_directory_removed (TrackerIndexingTree *indexing_tree,
2937 GFile *directory,
2938 gpointer user_data)
2939 {
2940 TrackerMinerFS *fs = user_data;
2941 TrackerMinerFSPrivate *priv = fs->priv;
2942 GTimer *timer = g_timer_new ();
2943
2944 /* Cancel all pending tasks on files inside the path given by file */
2945 tracker_task_pool_foreach (priv->task_pool,
2946 task_pool_cancel_foreach,
2947 directory);
2948
2949 g_debug (" Cancelled processing pool tasks at %f\n", g_timer_elapsed (timer, NULL));
2950
2951 tracker_task_pool_foreach (priv->writeback_pool,
2952 writeback_pool_cancel_foreach,
2953 directory);
2954
2955 g_debug (" Cancelled writeback pool tasks at %f\n",
2956 g_timer_elapsed (timer, NULL));
2957
2958 /* Remove anything contained in the removed directory
2959 * from all relevant processing queues.
2960 */
2961 tracker_priority_queue_foreach_remove (priv->items_updated,
2962 (GEqualFunc) file_equal_or_descendant,
2963 directory,
2964 (GDestroyNotify) g_object_unref);
2965 tracker_priority_queue_foreach_remove (priv->items_created,
2966 (GEqualFunc) file_equal_or_descendant,
2967 directory,
2968 (GDestroyNotify) g_object_unref);
2969
2970 g_debug (" Removed files at %f\n", g_timer_elapsed (timer, NULL));
2971
2972 g_message ("Finished remove directory operation in %f\n", g_timer_elapsed (timer, NULL));
2973 g_timer_destroy (timer);
2974 }
2975
2976 /**
2977 * tracker_miner_fs_directory_remove:
2978 * @fs: a #TrackerMinerFS
2979 * @file: #GFile for the directory to be removed
2980 *
2981 * Removes a directory from being inspected by @fs. Note that only directory
2982 * watches are removed.
2983 *
2984 * Returns: %TRUE if the directory was successfully removed.
2985 *
2986 * Since: 0.8
2987 **/
2988 gboolean
2989 tracker_miner_fs_directory_remove (TrackerMinerFS *fs,
2990 GFile *file)
2991 {
2992 TrackerMinerFSPrivate *priv;
2993
2994 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), FALSE);
2995 g_return_val_if_fail (G_IS_FILE (file), FALSE);
2996
2997 priv = fs->priv;
2998
2999 if (!tracker_indexing_tree_file_is_root (priv->indexing_tree, file)) {
3000 return FALSE;
3001 }
3002
3003 g_debug ("Removing directory");
3004 tracker_indexing_tree_remove (priv->indexing_tree, file);
3005
3006 return TRUE;
3007 }
3008
3009
3010 /**
3011 * tracker_miner_fs_directory_remove_full:
3012 * @fs: a #TrackerMinerFS
3013 * @file: #GFile for the directory to be removed
3014 *
3015 * Removes a directory from being inspected by @fs, and removes all
3016 * associated metadata of the directory (and its contents) from the
3017 * store.
3018 *
3019 * Returns: %TRUE if the directory was successfully removed.
3020 *
3021 * Since: 0.10
3022 **/
3023 gboolean
3024 tracker_miner_fs_directory_remove_full (TrackerMinerFS *fs,
3025 GFile *file)
3026 {
3027 TrackerDirectoryFlags flags;
3028
3029 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), FALSE);
3030 g_return_val_if_fail (G_IS_FILE (file), FALSE);
3031
3032 tracker_indexing_tree_get_root (fs->priv->indexing_tree, file, &flags);
3033
3034 if (tracker_miner_fs_directory_remove (fs, file)) {
3035 if ((flags & TRACKER_DIRECTORY_FLAG_PRESERVE) != 0) {
3036 /* If the preserve flag is unset, TrackerFileNotifier
3037 * will delete automatically files from this config
3038 * directory, if it's set, we force the delete here
3039 * to preserve remove_full() semantics.
3040 */
3041 trace_eq_push_tail ("DELETED", file, "on remove full");
3042 tracker_priority_queue_add (fs->priv->items_deleted,
3043 g_object_ref (file),
3044 G_PRIORITY_DEFAULT);
3045 item_queue_handlers_set_up (fs);
3046 }
3047
3048 return TRUE;
3049 }
3050
3051 return FALSE;
3052 }
3053
3054 static gboolean
3055 check_file_parents (TrackerMinerFS *fs,
3056 GFile *file)
3057 {
3058 GFile *parent, *root;
3059 GList *parents = NULL, *p;
3060
3061 parent = g_file_get_parent (file);
3062
3063 if (!parent) {
3064 return FALSE;
3065 }
3066
3067 root = tracker_indexing_tree_get_root (fs->priv->indexing_tree,
3068 parent, NULL);
3069 if (!root) {
3070 g_object_unref (parent);
3071 return FALSE;
3072 }
3073
3074 /* Add parent directories until we're past the config dir */
3075 while (parent &&
3076 !g_file_has_prefix (root, parent)) {
3077 parents = g_list_prepend (parents, parent);
3078 parent = g_file_get_parent (parent);
3079 }
3080
3081 /* Last parent fetched is not added to the list */
3082 if (parent) {
3083 g_object_unref (parent);
3084 }
3085
3086 for (p = parents; p; p = p->next) {
3087 trace_eq_push_tail ("UPDATED", p->data, "checking file parents");
3088 tracker_priority_queue_add (fs->priv->items_updated,
3089 p->data,
3090 G_PRIORITY_DEFAULT);
3091 }
3092
3093 g_list_free (parents);
3094
3095 return TRUE;
3096 }
3097
3098 /**
3099 * tracker_miner_fs_check_file_with_priority:
3100 * @fs: a #TrackerMinerFS
3101 * @file: #GFile for the file to check
3102 * @priority: the priority of the check task
3103 * @check_parents: whether to check parents and eligibility or not
3104 *
3105 * Tells the filesystem miner to check and index a file at
3106 * a given priority, this file must be part of the usual
3107 * crawling directories of #TrackerMinerFS. See
3108 * tracker_miner_fs_directory_add().
3109 *
3110 * Since: 0.10
3111 **/
3112 void
3113 tracker_miner_fs_check_file_with_priority (TrackerMinerFS *fs,
3114 GFile *file,
3115 gint priority,
3116 gboolean check_parents)
3117 {
3118 gboolean should_process = TRUE;
3119 gchar *path;
3120
3121 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3122 g_return_if_fail (G_IS_FILE (file));
3123
3124 if (check_parents) {
3125 should_process = should_check_file (fs, file, FALSE);
3126 }
3127
3128 path = g_file_get_path (file);
3129
3130 g_debug ("%s:'%s' (FILE) (requested by application)",
3131 should_process ? "Found " : "Ignored",
3132 path);
3133
3134 if (should_process) {
3135 if (check_parents && !check_file_parents (fs, file)) {
3136 return;
3137 }
3138
3139 trace_eq_push_tail ("UPDATED", file, "Requested by application");
3140 tracker_priority_queue_add (fs->priv->items_updated,
3141 g_object_ref (file),
3142 priority);
3143
3144 item_queue_handlers_set_up (fs);
3145 }
3146
3147 g_free (path);
3148 }
3149
3150
3151 /**
3152 * tracker_miner_fs_writeback_file:
3153 * @fs: a #TrackerMinerFS
3154 * @file: #GFile for the file to check
3155 * @rdf_types: A #GStrv with rdf types
3156 * @results: (element-type GStrv): A array of results from the preparation query
3157 *
3158 * Tells the filesystem miner to writeback a file.
3159 *
3160 * Since: 0.10.20
3161 **/
3162 void
3163 tracker_miner_fs_writeback_file (TrackerMinerFS *fs,
3164 GFile *file,
3165 GStrv rdf_types,
3166 GPtrArray *results)
3167 {
3168 gchar *path;
3169 ItemWritebackData *data;
3170
3171 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3172 g_return_if_fail (G_IS_FILE (file));
3173
3174 path = g_file_get_path (file);
3175
3176 g_debug ("Performing write-back:'%s' (requested by application)", path);
3177
3178 trace_eq_push_tail ("WRITEBACK", file, "Requested by application");
3179
3180 data = item_writeback_data_new (file, rdf_types, results);
3181 tracker_priority_queue_add (fs->priv->items_writeback, data,
3182 G_PRIORITY_DEFAULT);
3183
3184 item_queue_handlers_set_up (fs);
3185
3186 g_free (path);
3187 }
3188
3189 /**
3190 * tracker_miner_fs_writeback_notify:
3191 * @fs: a #TrackerMinerFS
3192 * @file: a #GFile
3193 * @error: a #GError with the error that happened during processing, or %NULL.
3194 *
3195 * Notifies @fs that all writing back on @file has been finished, if any error
3196 * happened during file data processing, it should be passed in @error, else
3197 * that parameter will contain %NULL to reflect success.
3198 *
3199 * Since: 0.10.20
3200 **/
3201 void
3202 tracker_miner_fs_writeback_notify (TrackerMinerFS *fs,
3203 GFile *file,
3204 const GError *error)
3205 {
3206 TrackerTask *task;
3207
3208 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3209 g_return_if_fail (G_IS_FILE (file));
3210
3211 fs->priv->total_files_notified++;
3212
3213 task = tracker_task_pool_find (fs->priv->writeback_pool, file);
3214
3215 if (!task) {
3216 gchar *uri;
3217
3218 uri = g_file_get_uri (file);
3219 g_critical ("%s has notified that file '%s' has been written back, "
3220 "but that file was not in the task pool. "
3221 "This is an implementation error, please ensure that "
3222 "tracker_miner_fs_writeback_notify() is called on the same "
3223 "GFile that is passed in ::writeback-file, and that this"
3224 "signal didn't return FALSE for it",
3225 G_OBJECT_TYPE_NAME (fs), uri);
3226 g_free (uri);
3227 } else if (error) {
3228
3229 if (!(error->domain == TRACKER_DBUS_ERROR &&
3230 error->code == TRACKER_DBUS_ERROR_UNSUPPORTED)) {
3231 g_warning ("Writeback operation failed: %s", error->message);
3232 }
3233
3234 /* We don't expect any further monitor
3235 * events on the original file.
3236 */
3237 tracker_task_pool_remove (fs->priv->writeback_pool, task);
3238 tracker_task_unref (task);
3239
3240 item_queue_handlers_set_up (fs);
3241 } else {
3242 ItemWritebackData *data;
3243
3244 data = tracker_task_get_data (task);
3245 data->notified = TRUE;
3246 }
3247
3248 /* Check monitor_item_updated_cb() for the remainder of this notify,
3249 * as the last event happening on the written back file would be an
3250 * UPDATED event caused by the changes on the cloned file, followed
3251 * by a MOVE onto the original file, so the delayed update happens
3252 * on the destination file.
3253 */
3254 }
3255
3256 /**
3257 * tracker_miner_fs_check_file:
3258 * @fs: a #TrackerMinerFS
3259 * @file: #GFile for the file to check
3260 * @check_parents: whether to check parents and eligibility or not
3261 *
3262 * Tells the filesystem miner to check and index a file,
3263 * this file must be part of the usual crawling directories
3264 * of #TrackerMinerFS. See tracker_miner_fs_directory_add().
3265 *
3266 * Since: 0.10
3267 **/
3268 void
3269 tracker_miner_fs_check_file (TrackerMinerFS *fs,
3270 GFile *file,
3271 gboolean check_parents)
3272 {
3273 tracker_miner_fs_check_file_with_priority (fs, file,
3274 G_PRIORITY_HIGH,
3275 check_parents);
3276 }
3277
3278 /**
3279 * tracker_miner_fs_check_directory_with_priority:
3280 * @fs: a #TrackerMinerFS
3281 * @file: #GFile for the directory to check
3282 * @priority: the priority of the check task
3283 * @check_parents: whether to check parents and eligibility or not
3284 *
3285 * Tells the filesystem miner to check and index a directory at
3286 * a given priority, this file must be part of the usual crawling
3287 * directories of #TrackerMinerFS. See tracker_miner_fs_directory_add().
3288 *
3289 * Since: 0.10
3290 **/
3291 void
3292 tracker_miner_fs_check_directory_with_priority (TrackerMinerFS *fs,
3293 GFile *file,
3294 gint priority,
3295 gboolean check_parents)
3296 {
3297 gboolean should_process = TRUE;
3298 gchar *path;
3299
3300 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3301 g_return_if_fail (G_IS_FILE (file));
3302
3303 if (check_parents) {
3304 should_process = should_check_file (fs, file, TRUE);
3305 }
3306
3307 path = g_file_get_path (file);
3308
3309 g_debug ("%s:'%s' (DIR) (requested by application)",
3310 should_process ? "Found " : "Ignored",
3311 path);
3312
3313 if (should_process) {
3314 if (check_parents && !check_file_parents (fs, file)) {
3315 return;
3316 }
3317
3318 /* FIXME: Apply priority */
3319 tracker_indexing_tree_add (fs->priv->indexing_tree,
3320 file,
3321 TRACKER_DIRECTORY_FLAG_RECURSE |
3322 TRACKER_DIRECTORY_FLAG_CHECK_MTIME |
3323 TRACKER_DIRECTORY_FLAG_MONITOR);
3324 }
3325
3326 g_free (path);
3327 }
3328
3329 /**
3330 * tracker_miner_fs_check_directory:
3331 * @fs: a #TrackerMinerFS
3332 * @file: #GFile for the directory to check
3333 * @check_parents: whether to check parents and eligibility or not
3334 *
3335 * Tells the filesystem miner to check and index a directory,
3336 * this file must be part of the usual crawling directories
3337 * of #TrackerMinerFS. See tracker_miner_fs_directory_add().
3338 *
3339 * Since: 0.10
3340 **/
3341 void
3342 tracker_miner_fs_check_directory (TrackerMinerFS *fs,
3343 GFile *file,
3344 gboolean check_parents)
3345 {
3346 tracker_miner_fs_check_directory_with_priority (fs, file,
3347 G_PRIORITY_DEFAULT,
3348 check_parents);
3349 }
3350
3351 /**
3352 * tracker_miner_fs_file_notify:
3353 * @fs: a #TrackerMinerFS
3354 * @file: a #GFile
3355 * @error: a #GError with the error that happened during processing, or %NULL.
3356 *
3357 * Notifies @fs that all processing on @file has been finished, if any error
3358 * happened during file data processing, it should be passed in @error, else
3359 * that parameter will contain %NULL to reflect success.
3360 *
3361 * Since: 0.8
3362 **/
3363 void
3364 tracker_miner_fs_file_notify (TrackerMinerFS *fs,
3365 GFile *file,
3366 const GError *error)
3367 {
3368 TrackerTask *task;
3369
3370 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3371 g_return_if_fail (G_IS_FILE (file));
3372
3373 fs->priv->total_files_notified++;
3374
3375 task = tracker_task_pool_find (fs->priv->task_pool, file);
3376
3377 if (!task) {
3378 gchar *uri;
3379
3380 uri = g_file_get_uri (file);
3381 g_critical ("%s has notified that file '%s' has been processed, "
3382 "but that file was not in the processing queue. "
3383 "This is an implementation error, please ensure that "
3384 "tracker_miner_fs_file_notify() is called on the same "
3385 "GFile that is passed in ::process-file, and that this"
3386 "signal didn't return FALSE for it",
3387 G_OBJECT_TYPE_NAME (fs), uri);
3388 g_free (uri);
3389
3390 if (item_queue_is_blocked_by_file (fs, file)) {
3391 /* Ensure we don't stall, although this is a very ugly situation */
3392 g_object_unref (fs->priv->item_queue_blocker);
3393 fs->priv->item_queue_blocker = NULL;
3394 item_queue_handlers_set_up (fs);
3395 }
3396
3397 return;
3398 }
3399
3400 item_add_or_update_cb (fs, task, error);
3401 }
3402
3403 /**
3404 * tracker_miner_fs_set_throttle:
3405 * @fs: a #TrackerMinerFS
3406 * @throttle: throttle value, between 0 and 1
3407 *
3408 * Tells the filesystem miner to throttle its operations.
3409 * a value of 0 means no throttling at all, so the miner
3410 * will perform operations at full speed, 1 is the slowest
3411 * value.
3412 *
3413 * Since: 0.8
3414 **/
3415 void
3416 tracker_miner_fs_set_throttle (TrackerMinerFS *fs,
3417 gdouble throttle)
3418 {
3419 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3420
3421 throttle = CLAMP (throttle, 0, 1);
3422
3423 if (fs->priv->throttle == throttle) {
3424 return;
3425 }
3426
3427 fs->priv->throttle = throttle;
3428
3429 /* Update timeouts */
3430 if (fs->priv->item_queues_handler_id != 0) {
3431 g_source_remove (fs->priv->item_queues_handler_id);
3432
3433 fs->priv->item_queues_handler_id =
3434 _tracker_idle_add (fs,
3435 item_queue_handlers_cb,
3436 fs);
3437 }
3438 }
3439
3440 /**
3441 * tracker_miner_fs_get_throttle:
3442 * @fs: a #TrackerMinerFS
3443 *
3444 * Gets the current throttle value. see tracker_miner_fs_set_throttle().
3445 *
3446 * Returns: current throttle value.
3447 *
3448 * Since: 0.8
3449 **/
3450 gdouble
3451 tracker_miner_fs_get_throttle (TrackerMinerFS *fs)
3452 {
3453 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), 0);
3454
3455 return fs->priv->throttle;
3456 }
3457
3458 /**
3459 * tracker_miner_fs_get_urn:
3460 * @fs: a #TrackerMinerFS
3461 * @file: a #GFile obtained in #TrackerMinerFS::process-file
3462 *
3463 * If the item exists in the store, this function retrieves
3464 * the URN for a #GFile being currently processed.
3465
3466 * If @file is not being currently processed by @fs, or doesn't
3467 * exist in the store yet, %NULL will be returned.
3468 *
3469 * Returns: (transfer none): The URN containing the data associated to @file,
3470 * or %NULL.
3471 *
3472 * Since: 0.8
3473 **/
3474 const gchar *
3475 tracker_miner_fs_get_urn (TrackerMinerFS *fs,
3476 GFile *file)
3477 {
3478 TrackerTask *task;
3479
3480 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
3481 g_return_val_if_fail (G_IS_FILE (file), NULL);
3482
3483 /* Check if found in currently processed data */
3484 task = tracker_task_pool_find (fs->priv->task_pool, file);
3485
3486 if (!task) {
3487 gchar *uri;
3488
3489 uri = g_file_get_uri (file);
3490
3491 g_critical ("File '%s' is not being currently processed, "
3492 "so the URN cannot be retrieved.", uri);
3493 g_free (uri);
3494
3495 return NULL;
3496 } else {
3497 UpdateProcessingTaskContext *ctxt;
3498
3499 /* We are only storing the URN in the created/updated tasks */
3500 ctxt = tracker_task_get_data (task);
3501
3502 if (!ctxt) {
3503 gchar *uri;
3504
3505 uri = g_file_get_uri (file);
3506 g_critical ("File '%s' is being processed, but not as a "
3507 "CREATED/UPDATED task, so cannot get URN",
3508 uri);
3509 g_free (uri);
3510 return NULL;
3511 }
3512
3513 return ctxt->urn;
3514 }
3515 }
3516
3517 /**
3518 * tracker_miner_fs_query_urn:
3519 * @fs: a #TrackerMinerFS
3520 * @file: a #GFile
3521 *
3522 * If the item exists in the store, this function retrieves
3523 * the URN of the given #GFile
3524
3525 * If @file doesn't exist in the store yet, %NULL will be returned.
3526 *
3527 * Returns: (transfer full): A newly allocated string with the URN containing the data associated
3528 * to @file, or %NULL.
3529 *
3530 * Since: 0.10
3531 **/
3532 gchar *
3533 tracker_miner_fs_query_urn (TrackerMinerFS *fs,
3534 GFile *file)
3535 {
3536 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
3537 g_return_val_if_fail (G_IS_FILE (file), NULL);
3538
3539 return g_strdup (tracker_file_notifier_get_file_iri (fs->priv->file_notifier,
3540 file));
3541 }
3542
3543 /**
3544 * tracker_miner_fs_get_parent_urn:
3545 * @fs: a #TrackerMinerFS
3546 * @file: a #GFile obtained in #TrackerMinerFS::process-file
3547 *
3548 * If @file is currently being processed by @fs, this function
3549 * will return the parent folder URN if any. This function is
3550 * useful to set the nie:belongsToContainer relationship. The
3551 * processing order of #TrackerMinerFS guarantees that a folder
3552 * has been already fully processed for indexing before any
3553 * children is processed, so most usually this function should
3554 * return non-%NULL.
3555 *
3556 * Returns: (transfer none): The parent folder URN, or %NULL.
3557 *
3558 * Since: 0.8
3559 **/
3560 const gchar *
3561 tracker_miner_fs_get_parent_urn (TrackerMinerFS *fs,
3562 GFile *file)
3563 {
3564 TrackerTask *task;
3565
3566 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
3567 g_return_val_if_fail (G_IS_FILE (file), NULL);
3568
3569 /* Check if found in currently processed data */
3570 task = tracker_task_pool_find (fs->priv->task_pool, file);
3571
3572 if (!task) {
3573 gchar *uri;
3574
3575 uri = g_file_get_uri (file);
3576
3577 g_critical ("File '%s' is not being currently processed, "
3578 "so the parent URN cannot be retrieved.", uri);
3579 g_free (uri);
3580
3581 return NULL;
3582 } else {
3583 UpdateProcessingTaskContext *ctxt;
3584
3585 /* We are only storing the URN in the created/updated tasks */
3586 ctxt = tracker_task_get_data (task);
3587
3588 if (!ctxt) {
3589 gchar *uri;
3590
3591 uri = g_file_get_uri (file);
3592 g_critical ("File '%s' is being processed, but not as a "
3593 "CREATED/UPDATED task, so cannot get parent "
3594 "URN",
3595 uri);
3596 g_free (uri);
3597 return NULL;
3598 }
3599
3600 return ctxt->parent_urn;
3601 }
3602 }
3603
3604 void
3605 tracker_miner_fs_force_recheck (TrackerMinerFS *fs)
3606 {
3607 #if 0
3608 GList *directories;
3609
3610 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3611
3612 g_message ("Forcing re-check on all index directories");
3613
3614 directories = fs->priv->config_directories;
3615
3616 while (directories) {
3617 tracker_priority_queue_add (fs->priv->directories,
3618 directory_data_ref (directories->data),
3619 G_PRIORITY_LOW);
3620 directories = directories->next;
3621 }
3622
3623 crawl_directories_start (fs);
3624 #endif
3625 }
3626
3627 /**
3628 * tracker_miner_fs_set_mtime_checking:
3629 * @fs: a #TrackerMinerFS
3630 * @mtime_checking: a #gboolean
3631 *
3632 * Tells the miner-fs that during the crawling phase, directory mtime
3633 * checks should or shouldn't be performed against the database to
3634 * make sure we have the most up to date version of the file being
3635 * checked at the time. Setting this to #FALSE can dramatically
3636 * improve the start up the crawling of the @fs.
3637 *
3638 * The down side is that using this consistently means that some files
3639 * on the disk may be out of date with files in the database.
3640 *
3641 * The main purpose of this function is for systems where a @fs is
3642 * running the entire time and where it is very unlikely that a file
3643 * could be changed outside between startup and shutdown of the
3644 * process using this API.
3645 *
3646 * The default if not set directly is that @mtime_checking is #TRUE.
3647 *
3648 * Since: 0.10
3649 **/
3650 void
3651 tracker_miner_fs_set_mtime_checking (TrackerMinerFS *fs,
3652 gboolean mtime_checking)
3653 {
3654 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3655
3656 fs->priv->mtime_checking = mtime_checking;
3657 }
3658
3659 /**
3660 * tracker_miner_fs_get_mtime_checking:
3661 * @fs: a #TrackerMinerFS
3662 *
3663 * Returns: #TRUE if mtime checks for directories against the database
3664 * are done when @fs crawls the file system, otherwise #FALSE.
3665 *
3666 * Since: 0.10
3667 **/
3668 gboolean
3669 tracker_miner_fs_get_mtime_checking (TrackerMinerFS *fs)
3670 {
3671 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), FALSE);
3672
3673 return fs->priv->mtime_checking;
3674 }
3675
3676 /**
3677 * tracker_miner_fs_force_mtime_checking:
3678 * @fs: a #TrackerMinerFS
3679 * @directory: a #GFile representing the directory
3680 *
3681 * Tells @fs to force mtime checking (regardless of the global mtime check
3682 * configuration) on the given @directory.
3683 *
3684 * Since: 0.12
3685 **/
3686 void
3687 tracker_miner_fs_force_mtime_checking (TrackerMinerFS *fs,
3688 GFile *directory)
3689 {
3690 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3691 g_return_if_fail (G_IS_FILE (directory));
3692
3693 tracker_indexing_tree_add (fs->priv->indexing_tree,
3694 directory,
3695 TRACKER_DIRECTORY_FLAG_CHECK_MTIME |
3696 TRACKER_DIRECTORY_FLAG_RECURSE |
3697 TRACKER_DIRECTORY_FLAG_MONITOR);
3698 }
3699
3700 void
3701 tracker_miner_fs_set_initial_crawling (TrackerMinerFS *fs,
3702 gboolean do_initial_crawling)
3703 {
3704 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3705
3706 fs->priv->initial_crawling = do_initial_crawling;
3707 }
3708
3709 gboolean
3710 tracker_miner_fs_get_initial_crawling (TrackerMinerFS *fs)
3711 {
3712 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), FALSE);
3713
3714 return fs->priv->initial_crawling;
3715 }
3716
3717 /**
3718 * tracker_miner_fs_has_items_to_process:
3719 * @fs: a #TrackerMinerFS
3720 *
3721 * Returns: #TRUE if there are items to process in the internal
3722 * queues, otherwise #FALSE.
3723 *
3724 * Since: 0.10
3725 **/
3726 gboolean
3727 tracker_miner_fs_has_items_to_process (TrackerMinerFS *fs)
3728 {
3729 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), FALSE);
3730
3731 if (tracker_file_notifier_is_active (fs->priv->file_notifier) ||
3732 !tracker_priority_queue_is_empty (fs->priv->items_deleted) ||
3733 !tracker_priority_queue_is_empty (fs->priv->items_created) ||
3734 !tracker_priority_queue_is_empty (fs->priv->items_updated) ||
3735 !tracker_priority_queue_is_empty (fs->priv->items_moved) ||
3736 !tracker_priority_queue_is_empty (fs->priv->items_writeback)) {
3737 return TRUE;
3738 }
3739
3740 return FALSE;
3741 }
3742
3743 /**
3744 * tracker_miner_fs_add_directory_without_parent:
3745 * @fs: a #TrackerMinerFS
3746 * @file: a #GFile
3747 *
3748 * Tells the miner-fs that the given #GFile corresponds to a
3749 * directory which was created in the store without a specific
3750 * parent object. In this case, when regenerating internal
3751 * caches, an extra query will be done so that these elements
3752 * are taken into account.
3753 *
3754 * Since: 0.10
3755 **/
3756 void
3757 tracker_miner_fs_add_directory_without_parent (TrackerMinerFS *fs,
3758 GFile *file)
3759 {
3760 g_return_if_fail (TRACKER_IS_MINER_FS (fs));
3761 g_return_if_fail (G_IS_FILE (file));
3762
3763 tracker_indexing_tree_add (fs->priv->indexing_tree,
3764 file,
3765 TRACKER_DIRECTORY_FLAG_RECURSE |
3766 TRACKER_DIRECTORY_FLAG_CHECK_MTIME |
3767 TRACKER_DIRECTORY_FLAG_MONITOR |
3768 TRACKER_DIRECTORY_FLAG_PRESERVE);
3769 }
3770
3771 /**
3772 * tracker_miner_fs_get_indexing_tree:
3773 * @fs: a #TrackerMinerFS
3774 *
3775 * Returns the #TrackerIndexingTree which determines
3776 * what files/directories are indexed by @fs
3777 *
3778 * Returns: (transfer none): The #TrackerIndexingTree
3779 * holding the indexing configuration
3780 **/
3781 TrackerIndexingTree *
3782 tracker_miner_fs_get_indexing_tree (TrackerMinerFS *fs)
3783 {
3784 g_return_val_if_fail (TRACKER_IS_MINER_FS (fs), NULL);
3785
3786 return fs->priv->indexing_tree;
3787 }
3788
3789 #ifdef EVENT_QUEUE_ENABLE_TRACE
3790
3791 static void
3792 trace_files_foreach (gpointer file,
3793 gpointer fs)
3794 {
3795 gchar *uri;
3796
3797 uri = g_file_get_uri (G_FILE (file));
3798 trace_eq ("(%s) '%s'",
3799 G_OBJECT_TYPE_NAME (G_OBJECT (fs)),
3800 uri);
3801 g_free (uri);
3802 }
3803
3804 static void
3805 trace_moved_foreach (gpointer moved_data,
3806 gpointer fs)
3807 {
3808 ItemMovedData *data = moved_data;
3809 gchar *source_uri;
3810 gchar *dest_uri;
3811
3812 source_uri = g_file_get_uri (data->source_file);
3813 dest_uri = g_file_get_uri (data->file);
3814 trace_eq ("(%s) '%s->%s'",
3815 G_OBJECT_TYPE_NAME (G_OBJECT (fs)),
3816 source_uri,
3817 dest_uri);
3818 g_free (source_uri);
3819 g_free (dest_uri);
3820 }
3821
3822 static void
3823 trace_writeback_foreach (gpointer writeback_data,
3824 gpointer fs)
3825 {
3826 ItemWritebackData *data = writeback_data;
3827 gchar *uri;
3828
3829 uri = g_file_get_uri (G_FILE (data->file));
3830 trace_eq ("(%s) '%s'",
3831 G_OBJECT_TYPE_NAME (G_OBJECT (fs)),
3832 uri);
3833 g_free (uri);
3834 }
3835
3836 static void
3837 miner_fs_trace_queue (TrackerMinerFS *fs,
3838 const gchar *queue_name,
3839 TrackerPriorityQueue *queue,
3840 GFunc foreach_cb)
3841 {
3842 trace_eq ("(%s) Queue '%s' has %u elements:",
3843 G_OBJECT_TYPE_NAME (fs),
3844 queue_name,
3845 tracker_priority_queue_get_length (queue));
3846 tracker_priority_queue_foreach (queue,
3847 foreach_cb,
3848 fs);
3849 }
3850
3851 static gboolean
3852 miner_fs_queues_status_trace_timeout_cb (gpointer data)
3853 {
3854 TrackerMinerFS *fs = data;
3855
3856 trace_eq ("(%s) ------------", G_OBJECT_TYPE_NAME (fs));
3857 miner_fs_trace_queue (fs, "CREATED", fs->priv->items_created, trace_files_foreach);
3858 miner_fs_trace_queue (fs, "UPDATED", fs->priv->items_updated, trace_files_foreach);
3859 miner_fs_trace_queue (fs, "DELETED", fs->priv->items_deleted, trace_files_foreach);
3860 miner_fs_trace_queue (fs, "MOVED", fs->priv->items_moved, trace_moved_foreach);
3861 miner_fs_trace_queue (fs, "WRITEBACK", fs->priv->items_writeback, trace_writeback_foreach);
3862
3863 return TRUE;
3864 }
3865
3866 #endif /* EVENT_QUEUE_ENABLE_TRACE */