KDirectoryContentsCounter: show intermediate dir size counting results, improve stopp...

Two user visible changes:
 * we can see the dir size changing as it is updated.
 * This makes the file counting a lot more reactive, when changing directories for instance.

`KDirectoryContentsCounterWorker::walkDir` is not recursive anymore.

The cache is now shared and only a single thread is used to count size recursively.

diff --git a/src/kitemviews/kfileitemlistview.cpp b/src/kitemviews/kfileitemlistview.cpp
index e2a27a5ea018bb0ec4d834ab2cd6da357caa9383..668ebdfb21f8ca5f3dafe29a59b83574147afb06 100644 (file)
--- a/src/kitemviews/kfileitemlistview.cpp
+++ b/src/kitemviews/kfileitemlistview.cpp
@@ -41,7 +41,6 @@ KFileItemListView::KFileItemListView(QGraphicsWidget *parent)
     , m_modelRolesUpdater(nullptr)
     , m_updateVisibleIndexRangeTimer(nullptr)
     , m_updateIconSizeTimer(nullptr)
     , m_modelRolesUpdater(nullptr)
     , m_updateVisibleIndexRangeTimer(nullptr)
     , m_updateIconSizeTimer(nullptr)

-    , m_scanDirectories(true)

 {
     setAcceptDrops(true);
 
 {
     setAcceptDrops(true);
 


@@ -119,19 +118,6 @@ qlonglong KFileItemListView::localFileSizePreviewLimit() const
     return m_modelRolesUpdater ? m_modelRolesUpdater->localFileSizePreviewLimit() : 0;
 }
 
     return m_modelRolesUpdater ? m_modelRolesUpdater->localFileSizePreviewLimit() : 0;
 }
 

-void KFileItemListView::setScanDirectories(bool enabled)
-{
-    m_scanDirectories = enabled;
-    if (m_modelRolesUpdater) {
-        m_modelRolesUpdater->setScanDirectories(m_scanDirectories);
-    }
-}
-
-bool KFileItemListView::scanDirectories()
-{
-    return m_scanDirectories;
-}
-

 QPixmap KFileItemListView::createDragPixmap(const KItemSet &indexes) const
 {
     if (!model()) {
 QPixmap KFileItemListView::createDragPixmap(const KItemSet &indexes) const
 {
     if (!model()) {


@@ -269,7 +255,6 @@ void KFileItemListView::onModelChanged(KItemModelBase *current, KItemModelBase *
     if (current) {
         m_modelRolesUpdater = new KFileItemModelRolesUpdater(static_cast<KFileItemModel *>(current), this);
         m_modelRolesUpdater->setIconSize(availableIconSize());
     if (current) {
         m_modelRolesUpdater = new KFileItemModelRolesUpdater(static_cast<KFileItemModel *>(current), this);
         m_modelRolesUpdater->setIconSize(availableIconSize());

-        m_modelRolesUpdater->setScanDirectories(scanDirectories());

 
         applyRolesToModel();
     }
 
         applyRolesToModel();
     }

diff --git a/src/kitemviews/kfileitemlistview.h b/src/kitemviews/kfileitemlistview.h
index 63bcf9e75378859322fbff344849b11d569498ad..b4be0093e7d2d675351b8c49805280639911d855 100644 (file)
--- a/src/kitemviews/kfileitemlistview.h
+++ b/src/kitemviews/kfileitemlistview.h
@@ -71,13 +71,6 @@ public:
     void setLocalFileSizePreviewLimit(qlonglong size);
     qlonglong localFileSizePreviewLimit() const;
 
     void setLocalFileSizePreviewLimit(qlonglong size);
     qlonglong localFileSizePreviewLimit() const;
 

-    /**
-     * If set to true, directories contents are scanned to determine their size
-     * Default true
-     */
-    void setScanDirectories(bool enabled);
-    bool scanDirectories();
-

     QPixmap createDragPixmap(const KItemSet &indexes) const override;
 
     /**
     QPixmap createDragPixmap(const KItemSet &indexes) const override;
 
     /**


@@ -137,7 +130,6 @@ private:
     KFileItemModelRolesUpdater *m_modelRolesUpdater;
     QTimer *m_updateVisibleIndexRangeTimer;
     QTimer *m_updateIconSizeTimer;
     KFileItemModelRolesUpdater *m_modelRolesUpdater;
     QTimer *m_updateVisibleIndexRangeTimer;
     QTimer *m_updateIconSizeTimer;

-    bool m_scanDirectories;

 
     friend class KFileItemListViewTest; // For unit testing
 };
 
     friend class KFileItemListViewTest; // For unit testing
 };

diff --git a/src/kitemviews/kfileitemlistwidget.cpp b/src/kitemviews/kfileitemlistwidget.cpp
index d9644bef5f8762bb1d1b6ba052817d57c397cc50..ec86833458cdef4b8a5099a1ebc4565ffd38ce4e 100644 (file)
--- a/src/kitemviews/kfileitemlistwidget.cpp
+++ b/src/kitemviews/kfileitemlistwidget.cpp
@@ -67,7 +67,7 @@ QString KFileItemListWidgetInformant::roleText(const QByteArray &role, const QHa
         if (values.value("isDir").toBool()) {
             if (!roleValue.isNull() && roleValue != -1) {
                 // The item represents a directory.
         if (values.value("isDir").toBool()) {
             if (!roleValue.isNull() && roleValue != -1) {
                 // The item represents a directory.

-                if (DetailsModeSettings::directorySizeCount()) {
+                if (DetailsModeSettings::directorySizeCount() || roleValue == -2 /* size is invalid */) {

                     //  Show the number of sub directories instead of the file size of the directory.
                     const int count = values.value("count").toInt();
                     text = i18ncp("@item:intable", "%1 item", "%1 items", count);
                     //  Show the number of sub directories instead of the file size of the directory.
                     const int count = values.value("count").toInt();
                     text = i18ncp("@item:intable", "%1 item", "%1 items", count);

diff --git a/src/kitemviews/kfileitemmodel.cpp b/src/kitemviews/kfileitemmodel.cpp
index 0d61775ad4586ea84d44df74692a5bd83ce264b5..6617fbec69fb9a74442aecce49ed489430fd9ef0 100644 (file)
--- a/src/kitemviews/kfileitemmodel.cpp
+++ b/src/kitemviews/kfileitemmodel.cpp
@@ -129,6 +129,8 @@ void KFileItemModel::refreshDirectory(const QUrl &url)
     }
 
     m_dirLister->openUrl(url, KDirLister::Reload);
     }
 
     m_dirLister->openUrl(url, KDirLister::Reload);

+
+    Q_EMIT directoryRefreshing();

 }
 
 QUrl KFileItemModel::directory() const
 }
 
 QUrl KFileItemModel::directory() const

diff --git a/src/kitemviews/kfileitemmodel.h b/src/kitemviews/kfileitemmodel.h
index afcd633b0458676ff8d61d29020b168d005bfc72..721569a0cc6fe40686502575e383606a92f76afd 100644 (file)
--- a/src/kitemviews/kfileitemmodel.h
+++ b/src/kitemviews/kfileitemmodel.h
@@ -217,6 +217,11 @@ Q_SIGNALS:
      */
     void directoryLoadingCompleted();
 
      */
     void directoryLoadingCompleted();
 

+    /**
+     * Is emitted when the model is being refreshed (F5 key press)
+     */
+    void directoryRefreshing();
+

     /**
      * Is emitted after the loading of a directory has been canceled.
      */
     /**
      * Is emitted after the loading of a directory has been canceled.
      */

diff --git a/src/kitemviews/kfileitemmodelrolesupdater.cpp b/src/kitemviews/kfileitemmodelrolesupdater.cpp
index c488fc0cf1be7a43ea5d1d0a9540b88ee79fd49a..6838d0861f78fb99a979bdaa4f5f0275183f626e 100644 (file)
--- a/src/kitemviews/kfileitemmodelrolesupdater.cpp
+++ b/src/kitemviews/kfileitemmodelrolesupdater.cpp
@@ -10,6 +10,7 @@
 #include "kfileitemmodel.h"
 #include "private/kdirectorycontentscounter.h"
 #include "private/kpixmapmodifier.h"
 #include "kfileitemmodel.h"
 #include "private/kdirectorycontentscounter.h"
 #include "private/kpixmapmodifier.h"

+#include "qdir.h"

 
 #include <KConfig>
 #include <KConfigGroup>
 
 #include <KConfig>
 #include <KConfigGroup>


@@ -20,6 +21,8 @@
 #include <KPluginMetaData>
 #include <KSharedConfig>
 
 #include <KPluginMetaData>
 #include <KSharedConfig>
 

+#include "dolphin_detailsmodesettings.h"
+

 #if HAVE_BALOO
 #include "private/kbaloorolesprovider.h"
 #include <Baloo/File>
 #if HAVE_BALOO
 #include "private/kbaloorolesprovider.h"
 #include <Baloo/File>


@@ -72,7 +75,6 @@ KFileItemModelRolesUpdater::KFileItemModelRolesUpdater(KFileItemModel *model, QO
     , m_resolvableRoles()
     , m_enabledPlugins()
     , m_localFileSizePreviewLimit(0)
     , m_resolvableRoles()
     , m_enabledPlugins()
     , m_localFileSizePreviewLimit(0)

-    , m_scanDirectories(true)

     , m_pendingSortRoleItems()
     , m_pendingIndexes()
     , m_pendingPreviewItems()
     , m_pendingSortRoleItems()
     , m_pendingIndexes()
     , m_pendingPreviewItems()


@@ -321,16 +323,6 @@ qlonglong KFileItemModelRolesUpdater::localFileSizePreviewLimit() const
     return m_localFileSizePreviewLimit;
 }
 
     return m_localFileSizePreviewLimit;
 }
 

-void KFileItemModelRolesUpdater::setScanDirectories(bool enabled)
-{
-    m_scanDirectories = enabled;
-}
-
-bool KFileItemModelRolesUpdater::scanDirectories() const
-{
-    return m_scanDirectories;
-}
-

 void KFileItemModelRolesUpdater::setHoverSequenceState(const QUrl &itemUrl, int seqIdx)
 {
     const KFileItem item = m_model->fileItem(itemUrl);
 void KFileItemModelRolesUpdater::setHoverSequenceState(const QUrl &itemUrl, int seqIdx)
 {
     const KFileItem item = m_model->fileItem(itemUrl);


@@ -423,8 +415,7 @@ void KFileItemModelRolesUpdater::slotItemsRemoved(const KItemRangeList &itemRang
         m_hoverSequenceLoadedItems.clear();
 
         killPreviewJob();
         m_hoverSequenceLoadedItems.clear();
 
         killPreviewJob();

-
-        if (m_scanDirectories) {
+        if (!m_model->showDirectoriesOnly()) {

             m_directoryContentsCounter->stopWorker();
         }
     } else {
             m_directoryContentsCounter->stopWorker();
         }
     } else {


@@ -856,10 +847,10 @@ void KFileItemModelRolesUpdater::applyChangedBalooRolesForItem(const KFileItem &
 #endif
 }
 
 #endif
 }
 

-void KFileItemModelRolesUpdater::slotDirectoryContentsCountReceived(const QString &path, int count, long size)
+void KFileItemModelRolesUpdater::slotDirectoryContentsCountReceived(const QString &path, int count, long long size)

 {
 {

-    const bool getSizeRole = m_roles.contains("size");

     const bool getIsExpandableRole = m_roles.contains("isExpandable");
     const bool getIsExpandableRole = m_roles.contains("isExpandable");

+    const bool getSizeRole = m_roles.contains("size");

 
     if (getSizeRole || getIsExpandableRole) {
         const int index = m_model->index(QUrl::fromLocalFile(path));
 
     if (getSizeRole || getIsExpandableRole) {
         const int index = m_model->index(QUrl::fromLocalFile(path));


@@ -1278,18 +1269,71 @@ bool KFileItemModelRolesUpdater::applyResolvedRoles(int index, ResolveHint hint)
 
 void KFileItemModelRolesUpdater::startDirectorySizeCounting(const KFileItem &item, int index)
 {
 
 void KFileItemModelRolesUpdater::startDirectorySizeCounting(const KFileItem &item, int index)
 {

-    if (item.isSlow()) {
+    if (DetailsModeSettings::directorySizeCount() || item.isSlow() || !item.isLocalFile()) {
+        // fastpath no recursion necessary
+
+        auto data = m_model->data(index);
+        if (data.value("size") == -2) {
+            // means job already started
+            return;
+        }
+
+        auto url = item.url();
+        if (!item.localPath().isEmpty()) {
+            // optimization for desktop:/, trash:/
+            url = QUrl::fromLocalFile(item.localPath());
+        }
+
+        data.insert("isExpandable", false);
+        data.insert("count", 0);
+        data.insert("size", -2); // invalid size, -1 means size unknown
+
+        disconnect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+        m_model->setData(index, data);
+        connect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+
+        auto listJob = KIO::listDir(url);
+        QObject::connect(listJob, &KIO::ListJob::entries, this, [this, index](const KJob * /*job*/, const KIO::UDSEntryList &list) {
+            auto data = m_model->data(index);
+            int origCount = data.value("count").toInt();
+            int entryCount = origCount;
+
+            for (const KIO::UDSEntry &entry : list) {
+                const auto name = entry.stringValue(KIO::UDSEntry::UDS_NAME);
+
+                if (name == QStringLiteral("..") || name == QStringLiteral(".")) {
+                    continue;
+                }
+                if (!m_model->showHiddenFiles() && name.startsWith(QLatin1Char('.'))) {
+                    continue;
+                }
+                if (m_model->showDirectoriesOnly() && !entry.isDir()) {
+                    continue;
+                }
+                ++entryCount;
+            }
+
+            // count has changed
+            if (origCount < entryCount) {
+                QHash<QByteArray, QVariant> data;
+                data.insert("isExpandable", entryCount > 0);
+                data.insert("count", entryCount);
+
+                disconnect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+                m_model->setData(index, data);
+                connect(m_model, &KFileItemModel::itemsChanged, this, &KFileItemModelRolesUpdater::slotItemsChanged);
+            }
+        });

         return;
     }
 
     // Tell m_directoryContentsCounter that we want to count the items
     // inside the directory. The result will be received in slotDirectoryContentsCountReceived.
         return;
     }
 
     // Tell m_directoryContentsCounter that we want to count the items
     // inside the directory. The result will be received in slotDirectoryContentsCountReceived.

-    if (m_scanDirectories && item.isLocalFile()) {
-        const QString path = item.localPath();
-        const auto priority = index >= m_firstVisibleIndex && index <= m_lastVisibleIndex ? KDirectoryContentsCounter::PathCountPriority::High
-                                                                                          : KDirectoryContentsCounter::PathCountPriority::Normal;
-        m_directoryContentsCounter->scanDirectory(path, priority);
-    }
+    const QString path = item.localPath();
+    const auto priority = index >= m_firstVisibleIndex && index <= m_lastVisibleIndex ? KDirectoryContentsCounter::PathCountPriority::High
+                                                                                      : KDirectoryContentsCounter::PathCountPriority::Normal;
+
+    m_directoryContentsCounter->scanDirectory(path, priority);

 }
 
 QHash<QByteArray, QVariant> KFileItemModelRolesUpdater::rolesData(const KFileItem &item, int index)
 }
 
 QHash<QByteArray, QVariant> KFileItemModelRolesUpdater::rolesData(const KFileItem &item, int index)

diff --git a/src/kitemviews/kfileitemmodelrolesupdater.h b/src/kitemviews/kfileitemmodelrolesupdater.h
index be6f2319311cbb56520be4a903f654193d6757b9..78fa757f82eba3c7847ccf7d5cd8cfb232eb3ac6 100644 (file)
--- a/src/kitemviews/kfileitemmodelrolesupdater.h
+++ b/src/kitemviews/kfileitemmodelrolesupdater.h
@@ -260,7 +260,7 @@ private Q_SLOTS:
     void applyChangedBalooRoles(const QString &file);
     void applyChangedBalooRolesForItem(const KFileItem &file);
 
     void applyChangedBalooRoles(const QString &file);
     void applyChangedBalooRolesForItem(const KFileItem &file);
 

-    void slotDirectoryContentsCountReceived(const QString &path, int count, long size);
+    void slotDirectoryContentsCountReceived(const QString &path, int count, long long size);

 
 private:
     /**
 
 private:
     /**


@@ -334,6 +334,9 @@ private:
     void trimHoverSequenceLoadedItems();
 
 private:
     void trimHoverSequenceLoadedItems();
 
 private:

+    /**
+     * enqueue directory size counting for KFileItem item at index
+     */

     void startDirectorySizeCounting(const KFileItem &item, int index);
 
     enum State { Idle, Paused, ResolvingSortRole, ResolvingAllRoles, PreviewJobRunning };
     void startDirectorySizeCounting(const KFileItem &item, int index);
 
     enum State { Idle, Paused, ResolvingSortRole, ResolvingAllRoles, PreviewJobRunning };


@@ -370,7 +373,6 @@ private:
     QSet<QByteArray> m_resolvableRoles;
     QStringList m_enabledPlugins;
     qulonglong m_localFileSizePreviewLimit;
     QSet<QByteArray> m_resolvableRoles;
     QStringList m_enabledPlugins;
     qulonglong m_localFileSizePreviewLimit;

-    bool m_scanDirectories;

 
     // Items for which the sort role still has to be determined.
     QSet<KFileItem> m_pendingSortRoleItems;
 
     // Items for which the sort role still has to be determined.
     QSet<KFileItem> m_pendingSortRoleItems;

diff --git a/src/kitemviews/private/kdirectorycontentscounter.cpp b/src/kitemviews/private/kdirectorycontentscounter.cpp
index 37e852ab9eea2b210b8196ae3dcd972bc609a8ef..e1a47419da6ac215c93c84d53d86033ee3b8e6bd 100644 (file)
--- a/src/kitemviews/private/kdirectorycontentscounter.cpp
+++ b/src/kitemviews/private/kdirectorycontentscounter.cpp
@@ -6,6 +6,7 @@
  */
 
 #include "kdirectorycontentscounter.h"
  */
 
 #include "kdirectorycontentscounter.h"

+#include "dolphin_detailsmodesettings.h"

 #include "kitemviews/kfileitemmodel.h"
 
 #include <KDirWatch>
 #include "kitemviews/kfileitemmodel.h"
 
 #include <KDirWatch>


@@ -16,36 +17,102 @@
 
 namespace
 {
 
 namespace
 {

+
+class LocalCache
+{
+public:
+    struct cacheData {
+        int count = 0;
+        long long size = 0;
+        ushort refCount = 0;
+        qint64 timestamp = 0;
+
+        inline operator bool() const
+        {
+            return timestamp != 0 && count != -1;
+        }
+    };
+
+    LocalCache()
+        : m_cache()
+    {
+    }
+
+    cacheData insert(const QString &key, cacheData data, bool inserted)
+    {
+        data.timestamp = QDateTime::currentMSecsSinceEpoch();
+        if (inserted) {
+            data.refCount += 1;
+        }
+        m_cache.insert(key, data);
+        return data;
+    }
+
+    cacheData value(const QString &key) const
+    {
+        return m_cache.value(key);
+    }
+
+    void unRefAll(const QSet<QString> &keys)
+    {
+        for (const auto &key : keys) {
+            auto entry = m_cache[key];
+            entry.refCount -= 1;
+            if (entry.refCount == 0) {
+                m_cache.remove(key);
+            }
+        }
+    }
+
+    void removeAll(const QSet<QString> &keys)
+    {
+        for (const auto &key : keys) {
+            m_cache.remove(key);
+        }
+    }
+
+private:
+    QHash<QString, cacheData> m_cache;
+};
+

 /// cache of directory counting result
 /// cache of directory counting result

-static QHash<QString, QPair<int, long>> *s_cache;
+static LocalCache *s_cache;
+static QThread *s_workerThread;
+static KDirectoryContentsCounterWorker *s_worker;

 }
 
 KDirectoryContentsCounter::KDirectoryContentsCounter(KFileItemModel *model, QObject *parent)
     : QObject(parent)
     , m_model(model)
 }
 
 KDirectoryContentsCounter::KDirectoryContentsCounter(KFileItemModel *model, QObject *parent)
     : QObject(parent)
     , m_model(model)

+    , m_priorityQueue()

     , m_queue()
     , m_queue()

-    , m_worker(nullptr)

     , m_workerIsBusy(false)
     , m_dirWatcher(nullptr)
     , m_watchedDirs()
 {
     , m_workerIsBusy(false)
     , m_dirWatcher(nullptr)
     , m_watchedDirs()
 {

-    connect(m_model, &KFileItemModel::itemsRemoved, this, &KDirectoryContentsCounter::slotItemsRemoved);
+    if (s_cache == nullptr) {
+        s_cache = new LocalCache();
+    }

 
 

-    if (!m_workerThread) {
-        m_workerThread = new QThread();
-        m_workerThread->start();
+    if (!s_workerThread) {
+        s_workerThread = new QThread();
+        s_workerThread->setObjectName(QStringLiteral("KDirectoryContentsCounterThread"));
+        s_workerThread->start();

     }
 
     }
 

-    if (s_cache == nullptr) {
-        s_cache = new QHash<QString, QPair<int, long>>();
+    if (!s_worker) {
+        s_worker = new KDirectoryContentsCounterWorker();
+        s_worker->moveToThread(s_workerThread);

     }
 
     }
 

-    m_worker = new KDirectoryContentsCounterWorker();
-    m_worker->moveToThread(m_workerThread);
+    connect(m_model, &KFileItemModel::itemsRemoved, this, &KDirectoryContentsCounter::slotItemsRemoved);
+    connect(m_model, &KFileItemModel::directoryRefreshing, this, &KDirectoryContentsCounter::slotDirectoryRefreshing);
+
+    connect(this, &KDirectoryContentsCounter::requestDirectoryContentsCount, s_worker, &KDirectoryContentsCounterWorker::countDirectoryContents);

 
 

-    connect(this, &KDirectoryContentsCounter::requestDirectoryContentsCount, m_worker, &KDirectoryContentsCounterWorker::countDirectoryContents);
-    connect(this, &KDirectoryContentsCounter::stop, m_worker, &KDirectoryContentsCounterWorker::stop);
-    connect(m_worker, &KDirectoryContentsCounterWorker::result, this, &KDirectoryContentsCounter::slotResult);
+    connect(s_worker, &KDirectoryContentsCounterWorker::result, this, &KDirectoryContentsCounter::slotResult);
+    connect(s_worker, &KDirectoryContentsCounterWorker::intermediateResult, this, &KDirectoryContentsCounter::result);
+    connect(s_worker, &KDirectoryContentsCounterWorker::finished, this, &KDirectoryContentsCounter::scheduleNext);

 
     m_dirWatcher = new KDirWatch(this);
     connect(m_dirWatcher, &KDirWatch::dirty, this, &KDirectoryContentsCounter::slotDirWatchDirty);
 
     m_dirWatcher = new KDirWatch(this);
     connect(m_dirWatcher, &KDirWatch::dirty, this, &KDirectoryContentsCounter::slotDirWatchDirty);


@@ -53,60 +120,20 @@ KDirectoryContentsCounter::KDirectoryContentsCounter(KFileItemModel *model, QObj
 
 KDirectoryContentsCounter::~KDirectoryContentsCounter()
 {
 
 KDirectoryContentsCounter::~KDirectoryContentsCounter()
 {

-    if (m_workerThread->isRunning()) {
-        // The worker thread will continue running. It could even be running
-        // a method of m_worker at the moment, so we delete it using
-        // deleteLater() to prevent a crash.
-        m_worker->deleteLater();
-    } else {
-        // There are no remaining workers -> stop the worker thread.
-        m_workerThread->quit();
-        m_workerThread->wait();
-        delete m_workerThread;
-        m_workerThread = nullptr;
-
-        // The worker thread has finished running now, so it's safe to delete
-        // m_worker. deleteLater() would not work at all because the event loop
-        // which would deliver the event to m_worker is not running any more.
-        delete m_worker;
-    }
+    s_cache->unRefAll(m_watchedDirs);

 }
 
 }
 

-void KDirectoryContentsCounter::slotResult(const QString &path, int count, long size)
+void KDirectoryContentsCounter::slotResult(const QString &path, int count, long long size)

 {
 {

-    m_workerIsBusy = false;
-
-    const QFileInfo info = QFileInfo(path);
-    const QString resolvedPath = info.canonicalFilePath();
-
-    if (!m_dirWatcher->contains(resolvedPath)) {
+    const auto fileInfo = QFileInfo(path);
+    const QString resolvedPath = fileInfo.canonicalFilePath();
+    if (fileInfo.isReadable() && !m_watchedDirs.contains(resolvedPath)) {

         m_dirWatcher->addDir(resolvedPath);
         m_dirWatcher->addDir(resolvedPath);

-        m_watchedDirs.insert(resolvedPath);
-    }
-
-    if (!m_priorityQueue.empty()) {
-        const QString firstPath = m_priorityQueue.front();
-        m_priorityQueue.pop_front();
-        scanDirectory(firstPath, PathCountPriority::High);
-    } else if (!m_queue.empty()) {
-        const QString firstPath = m_queue.front();
-        m_queue.pop_front();
-        scanDirectory(firstPath, PathCountPriority::Normal);

     }
     }

+    bool inserted = m_watchedDirs.insert(resolvedPath) == m_watchedDirs.end();

 
 

-    if (s_cache->contains(resolvedPath)) {
-        const auto pair = s_cache->value(resolvedPath);
-        if (pair.first == count && pair.second == size) {
-            // no change no need to send another result event
-            return;
-        }
-    }
-
-    if (info.dir().path() == m_model->rootItem().url().path()) {
-        // update cache or overwrite value
-        // when path is a direct children of the current model root
-        s_cache->insert(resolvedPath, QPair<int, long>(count, size));
-    }
+    // update cache or overwrite value
+    s_cache->insert(resolvedPath, {count, size, true}, inserted);

 
     // sends the results
     Q_EMIT result(path, count, size);
 
     // sends the results
     Q_EMIT result(path, count, size);


@@ -131,6 +158,11 @@ void KDirectoryContentsCounter::slotItemsRemoved()
 {
     const bool allItemsRemoved = (m_model->count() == 0);
 
 {
     const bool allItemsRemoved = (m_model->count() == 0);
 

+    if (allItemsRemoved) {
+        s_cache->removeAll(m_watchedDirs);
+        stopWorker();
+    }
+

     if (!m_watchedDirs.isEmpty()) {
         // Don't let KDirWatch watch for removed items
         if (allItemsRemoved) {
     if (!m_watchedDirs.isEmpty()) {
         // Don't let KDirWatch watch for removed items
         if (allItemsRemoved) {


@@ -138,7 +170,6 @@ void KDirectoryContentsCounter::slotItemsRemoved()
                 m_dirWatcher->removeDir(path);
             }
             m_watchedDirs.clear();
                 m_dirWatcher->removeDir(path);
             }
             m_watchedDirs.clear();

-            m_queue.clear();

         } else {
             QMutableSetIterator<QString> it(m_watchedDirs);
             while (it.hasNext()) {
         } else {
             QMutableSetIterator<QString> it(m_watchedDirs);
             while (it.hasNext()) {


@@ -152,46 +183,102 @@ void KDirectoryContentsCounter::slotItemsRemoved()
     }
 }
 
     }
 }
 

-void KDirectoryContentsCounter::scanDirectory(const QString &path, PathCountPriority priority)
+void KDirectoryContentsCounter::slotDirectoryRefreshing()
+{
+    s_cache->removeAll(m_watchedDirs);
+}
+
+void KDirectoryContentsCounter::scheduleNext()

 {
 {

-    const QString resolvedPath = QFileInfo(path).canonicalFilePath();
-    const bool alreadyInCache = s_cache->contains(resolvedPath);
-    if (alreadyInCache) {
+    if (!m_priorityQueue.empty()) {
+        m_currentPath = m_priorityQueue.front();
+        m_priorityQueue.pop_front();
+    } else if (!m_queue.empty()) {
+        m_currentPath = m_queue.front();
+        m_queue.pop_front();
+    } else {
+        m_currentPath.clear();
+        m_workerIsBusy = false;
+        return;
+    }
+
+    const auto fileInfo = QFileInfo(m_currentPath);
+    const QString resolvedPath = fileInfo.canonicalFilePath();
+    const auto pair = s_cache->value(resolvedPath);
+    if (pair) {

         // fast path when in cache
         // will be updated later if result has changed
         // fast path when in cache
         // will be updated later if result has changed

-        const auto pair = s_cache->value(resolvedPath);
-        Q_EMIT result(path, pair.first, pair.second);
-    }
-
-    if (m_workerIsBusy) {
-        // only enqueue path not yet in queue
-        if (std::find(m_queue.begin(), m_queue.end(), path) == m_queue.end()
-            && std::find(m_priorityQueue.begin(), m_priorityQueue.end(), path) == m_priorityQueue.end()) {
-            if (priority == PathCountPriority::Normal) {
-                if (alreadyInCache) {
-                    // if we already knew the dir size, it gets lower priority
-                    m_queue.push_back(path);
-                } else {
-                    m_queue.push_front(path);
-                }
-            } else {
-                // append to priority queue
-                m_priorityQueue.push_back(path);
-            }
-        }
-    } else {
-        KDirectoryContentsCounterWorker::Options options;
+        Q_EMIT result(m_currentPath, pair.count, pair.size);
+    }
+
+    // if scanned fully recently, skip rescan
+    if (pair && pair.timestamp >= fileInfo.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch()) {
+        scheduleNext();
+        return;
+    }
+
+    KDirectoryContentsCounterWorker::Options options;
+
+    if (m_model->showHiddenFiles()) {
+        options |= KDirectoryContentsCounterWorker::CountHiddenFiles;
+    }
+
+    m_workerIsBusy = true;
+    Q_EMIT requestDirectoryContentsCount(m_currentPath, options, DetailsModeSettings::recursiveDirectorySizeLimit());
+}

 
 

-        if (m_model->showHiddenFiles()) {
-            options |= KDirectoryContentsCounterWorker::CountHiddenFiles;
+void KDirectoryContentsCounter::enqueuePathScanning(const QString &path, bool alreadyInCache, PathCountPriority priority)
+{
+    // ensure to update the entry in the queue
+    auto it = std::find(m_queue.begin(), m_queue.end(), path);
+    if (it != m_queue.end()) {
+        m_queue.erase(it);
+    } else {
+        it = std::find(m_priorityQueue.begin(), m_priorityQueue.end(), path);
+        if (it != m_priorityQueue.end()) {
+            m_priorityQueue.erase(it);

         }
         }

+    }

 
 

-        if (m_model->showDirectoriesOnly()) {
-            options |= KDirectoryContentsCounterWorker::CountDirectoriesOnly;
+    if (priority == PathCountPriority::Normal) {
+        if (alreadyInCache) {
+            // we already knew the dir size
+            // otherwise it gets lower priority
+            m_queue.push_back(path);
+        } else {
+            m_queue.push_front(path);

         }
         }

+    } else {
+        // append to priority queue
+        m_priorityQueue.push_front(path);
+    }
+}
+
+void KDirectoryContentsCounter::scanDirectory(const QString &path, PathCountPriority priority)
+{
+    if (m_workerIsBusy && m_currentPath == path) {
+        // already listing
+        return;
+    }
+
+    const auto fileInfo = QFileInfo(path);
+    const QString resolvedPath = fileInfo.canonicalFilePath();
+    const auto pair = s_cache->value(resolvedPath);
+    if (pair) {
+        // fast path when in cache
+        // will be updated later if result has changed
+        Q_EMIT result(path, pair.count, pair.size);
+    }

 
 

-        Q_EMIT requestDirectoryContentsCount(path, options);
-        m_workerIsBusy = true;
+    // if scanned fully recently, skip rescan
+    if (pair && pair.timestamp >= fileInfo.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch()) {
+        return;
+    }
+
+    enqueuePathScanning(path, pair, priority);
+
+    if (!m_workerIsBusy && !s_worker->stopping()) {
+        scheduleNext();

     }
 }
 
     }
 }
 


@@ -199,7 +286,9 @@ void KDirectoryContentsCounter::stopWorker()
 {
     m_queue.clear();
     m_priorityQueue.clear();
 {
     m_queue.clear();
     m_priorityQueue.clear();

-    Q_EMIT stop();
-}

 
 

-QThread *KDirectoryContentsCounter::m_workerThread = nullptr;
+    if (m_workerIsBusy && m_currentPath == s_worker->scannedPath()) {
+        s_worker->stop();
+    }
+    m_currentPath.clear();
+}

diff --git a/src/kitemviews/private/kdirectorycontentscounter.h b/src/kitemviews/private/kdirectorycontentscounter.h
index 9a5e4a86b333f1dbb156a3071abc3c6048d2d6c9..552b5560eba9be366ce7490036d6f713eefdba31 100644 (file)
--- a/src/kitemviews/private/kdirectorycontentscounter.h
+++ b/src/kitemviews/private/kdirectorycontentscounter.h
@@ -49,32 +49,32 @@ Q_SIGNALS:
      * Signals that the directory \a path contains \a count items of size \a
      * Size calculation depends on parameter DetailsModeSettings::recursiveDirectorySizeLimit
      */
      * Signals that the directory \a path contains \a count items of size \a
      * Size calculation depends on parameter DetailsModeSettings::recursiveDirectorySizeLimit
      */

-    void result(const QString &path, int count, long size);
+    void result(const QString &path, int count, long long size);

 
 

-    void requestDirectoryContentsCount(const QString &path, KDirectoryContentsCounterWorker::Options options);
-
-    void stop();
+    void requestDirectoryContentsCount(const QString &path, KDirectoryContentsCounterWorker::Options options, int maxRecursiveLevel);

 
 private Q_SLOTS:
 
 private Q_SLOTS:

-    void slotResult(const QString &path, int count, long size);
+    void slotResult(const QString &path, int count, long long size);

     void slotDirWatchDirty(const QString &path);
     void slotItemsRemoved();
     void slotDirWatchDirty(const QString &path);
     void slotItemsRemoved();

+    void slotDirectoryRefreshing();
+    void scheduleNext();

 
 private:
 
 private:

+    void enqueuePathScanning(const QString &path, bool alreadyInCache, PathCountPriority priority);
+

     KFileItemModel *m_model;
 
     // Used as FIFO queues.
     std::list<QString> m_priorityQueue;
     std::list<QString> m_queue;
 
     KFileItemModel *m_model;
 
     // Used as FIFO queues.
     std::list<QString> m_priorityQueue;
     std::list<QString> m_queue;
 

-    static QThread *m_workerThread;
-
-    KDirectoryContentsCounterWorker *m_worker;

     bool m_workerIsBusy;
 
     KDirWatch *m_dirWatcher;
     QSet<QString> m_watchedDirs; // Required as sadly KDirWatch does not offer a getter method
                                  // to get all watched directories.
     bool m_workerIsBusy;
 
     KDirWatch *m_dirWatcher;
     QSet<QString> m_watchedDirs; // Required as sadly KDirWatch does not offer a getter method
                                  // to get all watched directories.

+    QString m_currentPath;

 };
 
 #endif
 };
 
 #endif

diff --git a/src/kitemviews/private/kdirectorycontentscounterworker.cpp b/src/kitemviews/private/kdirectorycontentscounterworker.cpp
index 2227ebbc262953fb7ad7d1373ebde61282232c99..eb456da254b1376cd6558a63e14c00fa431dba43 100644 (file)
--- a/src/kitemviews/private/kdirectorycontentscounterworker.cpp
+++ b/src/kitemviews/private/kdirectorycontentscounterworker.cpp
@@ -7,16 +7,16 @@
 
 #include "kdirectorycontentscounterworker.h"
 
 
 #include "kdirectorycontentscounterworker.h"
 

-// Required includes for subItemsCount():
+// Required includes for countDirectoryContents():

 #ifdef Q_OS_WIN
 #include <QDir>
 #else
 #ifdef Q_OS_WIN
 #include <QDir>
 #else

-#include <QFile>
-#include <qplatformdefs.h>
+#include <QElapsedTimer>
+#include <fts.h>
+#include <sys/stat.h>
+#include <sys/types.h>

 #endif
 
 #endif
 

-#include "dolphin_detailsmodesettings.h"
-

 KDirectoryContentsCounterWorker::KDirectoryContentsCounterWorker(QObject *parent)
     : QObject(parent)
 {
 KDirectoryContentsCounterWorker::KDirectoryContentsCounterWorker(QObject *parent)
     : QObject(parent)
 {


@@ -24,100 +24,135 @@ KDirectoryContentsCounterWorker::KDirectoryContentsCounterWorker(QObject *parent
 }
 
 #ifndef Q_OS_WIN
 }
 
 #ifndef Q_OS_WIN

-KDirectoryContentsCounterWorker::CountResult
-KDirectoryContentsCounterWorker::walkDir(const QString &dirPath, const bool countHiddenFiles, const bool countDirectoriesOnly, const uint allowedRecursiveLevel)
+void KDirectoryContentsCounterWorker::walkDir(const QString &dirPath, bool countHiddenFiles, uint allowedRecursiveLevel)

 {
 {

-    int count = -1;
-    long size = -1;
-    auto dir = QT_OPENDIR(QFile::encodeName(dirPath));
-    if (dir) {
-        count = 0;
-        size = 0;
-        QT_DIRENT *dirEntry;
-        QT_STATBUF buf;
-
-        while (!m_stopping && (dirEntry = QT_READDIR(dir))) {
-            if (dirEntry->d_name[0] == '.') {
-                if (dirEntry->d_name[1] == '\0' || !countHiddenFiles) {
-                    // Skip "." or hidden files
-                    continue;
-                }
-                if (dirEntry->d_name[1] == '.' && dirEntry->d_name[2] == '\0') {
-                    // Skip ".."
-                    continue;
-                }
+    QByteArray text = dirPath.toLocal8Bit();
+    char *rootPath = new char[text.size() + 1];
+    ::strncpy(rootPath, text.constData(), text.size() + 1);
+    char *path[2]{rootPath, nullptr};
+
+    // follow symlink only for root dir
+    auto tree = ::fts_open(path, FTS_COMFOLLOW | FTS_PHYSICAL | FTS_XDEV, nullptr);
+    if (!tree) {
+        delete[] rootPath;
+        return;
+    }
+
+    FTSENT *node;
+    long long totalSize = -1;
+    int totalCount = -1;
+    QElapsedTimer timer;
+    timer.start();
+
+    while ((node = fts_read(tree)) && !m_stopping) {
+        auto info = node->fts_info;
+
+        if (info == FTS_DC) {
+            // ignore directories clausing cycles
+            continue;
+        }
+        if (info == FTS_DNR) {
+            // ignore directories that can’t be read
+            continue;
+        }
+        if (info == FTS_ERR) {
+            // ignore directories causing errors
+            fts_set(tree, node, FTS_SKIP);
+            continue;
+        }
+        if (info == FTS_DP) {
+            // ignore end traversal of dir
+            continue;
+        }
+
+        if (!countHiddenFiles && node->fts_name[0] == '.' && strncmp(".git", node->fts_name, 4) != 0) {
+            // skip hidden files, except .git dirs
+            if (info == FTS_D) {
+                fts_set(tree, node, FTS_SKIP);
+            }
+            continue;
+        }
+
+        if (info == FTS_F) {
+            // only count files that are physical (aka skip /proc/kcore...)
+            // naive size counting not taking into account effective disk space used (aka size/block_size * block_size)
+            // skip directory size (usually a 4KB block)
+            if (node->fts_statp->st_blocks > 0) {
+                totalSize += node->fts_statp->st_size;

             }
             }

+        }

 
 

-            // If only directories are counted, consider an unknown file type and links also
-            // as directory instead of trying to do an expensive stat()
-            // (see bugs 292642 and 299997).
-            const bool countEntry = !countDirectoriesOnly || dirEntry->d_type == DT_DIR || dirEntry->d_type == DT_LNK || dirEntry->d_type == DT_UNKNOWN;
-            if (countEntry) {
-                ++count;
+        if (info == FTS_D) {
+            if (node->fts_level == 0) {
+                // first read was sucessful, we can init counters
+                totalSize = 0;
+                totalCount = 0;

             }
 
             }
 

-            if (allowedRecursiveLevel > 0) {
-                QString nameBuf = QStringLiteral("%1/%2").arg(dirPath, dirEntry->d_name);
-
-                if (dirEntry->d_type == DT_REG) {
-                    if (QT_STAT(nameBuf.toLocal8Bit(), &buf) == 0) {
-                        size += buf.st_size;
-                    }
-                }
-                if (!m_stopping && dirEntry->d_type == DT_DIR) {
-                    // recursion for dirs
-                    auto subdirResult = walkDir(nameBuf, countHiddenFiles, countDirectoriesOnly, allowedRecursiveLevel - 1);
-                    if (subdirResult.size > 0) {
-                        size += subdirResult.size;
-                    }
-                }
+            if (node->fts_level > (int)allowedRecursiveLevel) {
+                // skip too deep nodes
+                fts_set(tree, node, FTS_SKIP);
+                continue;

             }
         }
             }
         }

-        QT_CLOSEDIR(dir);
+        // count first level elements
+        if (node->fts_level == 1) {
+            ++totalCount;
+        }
+
+        // delay intermediate results
+        if (timer.hasExpired(200) || node->fts_level == 0) {
+            Q_EMIT intermediateResult(dirPath, totalCount, totalSize);
+            timer.restart();
+        }
+    }
+
+    delete[] rootPath;
+    fts_close(tree);
+    if (errno != 0) {
+        return;
+    }
+
+    if (!m_stopping) {
+        Q_EMIT result(dirPath, totalCount, totalSize);

     }
     }

-    return KDirectoryContentsCounterWorker::CountResult{count, size};

 }
 #endif
 
 }
 #endif
 

-KDirectoryContentsCounterWorker::CountResult KDirectoryContentsCounterWorker::subItemsCount(const QString &path, Options options)
+void KDirectoryContentsCounterWorker::stop()
+{
+    m_stopping = true;
+}
+
+bool KDirectoryContentsCounterWorker::stopping() const
+{
+    return m_stopping;
+}
+
+QString KDirectoryContentsCounterWorker::scannedPath() const
+{
+    return m_scannedPath;
+}
+
+void KDirectoryContentsCounterWorker::countDirectoryContents(const QString &path, Options options, int maxRecursiveLevel)

 {
     const bool countHiddenFiles = options & CountHiddenFiles;
 {
     const bool countHiddenFiles = options & CountHiddenFiles;

-    const bool countDirectoriesOnly = options & CountDirectoriesOnly;

 
 #ifdef Q_OS_WIN
     QDir dir(path);
 
 #ifdef Q_OS_WIN
     QDir dir(path);

-    QDir::Filters filters = QDir::NoDotAndDotDot | QDir::System;
+    QDir::Filters filters = QDir::NoDotAndDotDot | QDir::System | QDir::AllEntries;

     if (countHiddenFiles) {
         filters |= QDir::Hidden;
     }
     if (countHiddenFiles) {
         filters |= QDir::Hidden;
     }

-    if (countDirectoriesOnly) {
-        filters |= QDir::Dirs;
-    } else {
-        filters |= QDir::AllEntries;
-    }
-    return {static_cast<int>(dir.entryList(filters).count()), 0};
-#else

 
 

-    const uint maxRecursiveLevel = DetailsModeSettings::directorySizeCount() ? 1 : DetailsModeSettings::recursiveDirectorySizeLimit();
+    Q_EMIT result(path, static_cast<int>(dir.entryList(filters).count()), 0);
+#else

 
 

-    auto res = walkDir(path, countHiddenFiles, countDirectoriesOnly, maxRecursiveLevel);
+    m_scannedPath = path;
+    walkDir(path, countHiddenFiles, maxRecursiveLevel);

 
 

-    return res;

 #endif
 #endif

-}
-
-void KDirectoryContentsCounterWorker::stop()
-{
-    m_stopping = true;
-}

 
 

-void KDirectoryContentsCounterWorker::countDirectoryContents(const QString &path, Options options)
-{

     m_stopping = false;
     m_stopping = false;

-
-    auto res = subItemsCount(path, options);
-
-    if (!m_stopping) {
-        Q_EMIT result(path, res.count, res.size);
-    }
+    Q_EMIT finished();

 }
 }

diff --git a/src/kitemviews/private/kdirectorycontentscounterworker.h b/src/kitemviews/private/kdirectorycontentscounterworker.h
index 5266960cd39f88438de177d4bf258de45a835d27..077df9f697ffbf1682dd6a85b9bb3cc227b86ae5 100644 (file)
--- a/src/kitemviews/private/kdirectorycontentscounterworker.h
+++ b/src/kitemviews/private/kdirectorycontentscounterworker.h
@@ -17,32 +17,20 @@ class KDirectoryContentsCounterWorker : public QObject
     Q_OBJECT
 
 public:
     Q_OBJECT
 
 public:

-    enum Option { NoOptions = 0x0, CountHiddenFiles = 0x1, CountDirectoriesOnly = 0x2 };
+    enum Option { NoOptions = 0x0, CountHiddenFiles = 0x1 };

     Q_DECLARE_FLAGS(Options, Option)
 
     Q_DECLARE_FLAGS(Options, Option)
 

-    struct CountResult {
-        /// number of elements in the directory
-        int count;
-        /// Recursive sum of the size of the directory content files and folders
-        /// Calculation depends on DetailsModeSettings::recursiveDirectorySizeLimit
-        long size;
-    };
-

     explicit KDirectoryContentsCounterWorker(QObject *parent = nullptr);
 
     explicit KDirectoryContentsCounterWorker(QObject *parent = nullptr);
 

-    /**
-     * Counts the items inside the directory \a path using the options
-     * \a options.
-     *
-     * @return The number of items.
-     */
-    CountResult subItemsCount(const QString &path, Options options);
-
+    bool stopping() const;
+    QString scannedPath() const;

 Q_SIGNALS:
     /**
      * Signals that the directory \a path contains \a count items and optionally the size of its content.
      */
 Q_SIGNALS:
     /**
      * Signals that the directory \a path contains \a count items and optionally the size of its content.
      */

-    void result(const QString &path, int count, long size);
+    void result(const QString &path, int count, long long size);
+    void intermediateResult(const QString &path, int count, long long size);
+    void finished();

 
 public Q_SLOTS:
     /**
 
 public Q_SLOTS:
     /**


@@ -52,16 +40,17 @@ public Q_SLOTS:
     // Note that the full type name KDirectoryContentsCounterWorker::Options
     // is needed here. Just using 'Options' is OK for the compiler, but
     // confuses moc.
     // Note that the full type name KDirectoryContentsCounterWorker::Options
     // is needed here. Just using 'Options' is OK for the compiler, but
     // confuses moc.

-    void countDirectoryContents(const QString &path, KDirectoryContentsCounterWorker::Options options);
+    void countDirectoryContents(const QString &path, KDirectoryContentsCounterWorker::Options options, int maxRecursiveLevel);

     void stop();
 
 private:
 #ifndef Q_OS_WIN
     void stop();
 
 private:
 #ifndef Q_OS_WIN

-    KDirectoryContentsCounterWorker::CountResult
-    walkDir(const QString &dirPath, const bool countHiddenFiles, const bool countDirectoriesOnly, const uint allowedRecursiveLevel);
+    void walkDir(const QString &dirPath, bool countHiddenFiles, uint allowedRecursiveLevel);

 #endif
 
 #endif
 

-    bool m_stopping = false;
+    QString m_scannedPath;
+
+    std::atomic<bool> m_stopping = false;

 };
 
 Q_DECLARE_METATYPE(KDirectoryContentsCounterWorker::Options)
 };
 
 Q_DECLARE_METATYPE(KDirectoryContentsCounterWorker::Options)

diff --git a/src/panels/folders/folderspanel.cpp b/src/panels/folders/folderspanel.cpp
index 914692000018f8d9b6161667b635db75b1a114c5..f304654cbfe77e08fc7418fb6e01f35f96b08810 100644 (file)
--- a/src/panels/folders/folderspanel.cpp
+++ b/src/panels/folders/folderspanel.cpp
@@ -131,7 +131,6 @@ void FoldersPanel::showEvent(QShowEvent *event)
         // This assures that no performance and memory overhead is given when the folders panel is not
         // used at all and stays invisible.
         KFileItemListView *view = new KFileItemListView();
         // This assures that no performance and memory overhead is given when the folders panel is not
         // used at all and stays invisible.
         KFileItemListView *view = new KFileItemListView();

-        view->setScanDirectories(false);

         view->setWidgetCreator(new KItemListWidgetCreator<FoldersItemListWidget>());
         view->setSupportsItemExpanding(true);
         // Set the opacity to 0 initially. The opacity will be increased after the loading of the initial tree
         view->setWidgetCreator(new KItemListWidgetCreator<FoldersItemListWidget>());
         view->setSupportsItemExpanding(true);
         // Set the opacity to 0 initially. The opacity will be increased after the loading of the initial tree