[Companion] SD Sync reworked: (#4906)

* Complete re-write of SyncProcess, now more robust, UI-independent, runs in own thread, large speed boost;
    * Use file hash comparisons instead of byte-level compare with string methods (probable fixes for #4837 and #4848);
    * New pre-sync options dialog with selectable folders and other settings (settings are remembered until app exit);
    * Option to choose sync direction(s) (up/down/bi) and the order in which bi-directional sync runs;
    * Option to choose file comparison mode (timestamp/hash/both) or force overwrite (for one-way sync);
    * Option to set a maximum file size to copy;
    * Option to do a test run w/out actually copying any files;
    * Better reporting of progress, total counts (copied/updated/skipped), and errors, with color coding;
    * Much more reliable to cancel long-running/hung copy process;
    * ProgressWidget can now display rich-text in the text browser, added some new functions for usage;
    * Misc. small layout tweaks to ProgressDialog and ProgressWidget to make them behave/look better;

companion/src/mainwindow.cpp

@@ -44,6 +44,7 @@
 #include "process_sync.h"
 #include "radiointerface.h"
 #include "progressdialog.h"
+#include "progresswidget.h"
 #include "storage.h"
 #include "translations.h"
 
@@ -735,24 +736,254 @@ void MainWindow::contributors()
   dialog->deleteLater();
 }
 
+// Create a widget with a line edit and folder select button and handles all interactions. Features autosuggest
+//   path hints while typing, invalid paths shown in red. The label string is only for dialog title, not a QLabel.
+// This should probably be moved some place more reusable, esp. the QFileSystemModel.
+QWidget * folderSelectorWidget(QString * path, const QString & label, QWidget * parent)
+{
+  static QFileSystemModel fileModel;
+  static bool init = false;
+  if (!init) {
+    init = true;
+    fileModel.setFilter(QDir::Dirs);
+    fileModel.setRootPath("/");
+  }
+
+  QWidget * fsw = new QWidget(parent);
+  QLineEdit * le = new QLineEdit(parent);
+  QCompleter * fsc = new QCompleter(fsw);
+  fsc->setModel(&fileModel);
+  //fsc->setCompletionMode(QCompleter::InlineCompletion);
+  le->setCompleter(fsc);
+
+  QToolButton * btn = new QToolButton(fsw);
+  btn->setIcon(CompanionIcon("open.png"));
+  QHBoxLayout * l = new QHBoxLayout(fsw);
+  l->setContentsMargins(0,0,0,0);
+  l->setSpacing(3);
+  l->addWidget(le);
+  l->addWidget(btn);
+
+  QObject::connect(btn, &QToolButton::clicked, [=]() {
+    QString dir = QFileDialog::getExistingDirectory(parent, label, le->text(), 0);
+    if (!dir.isEmpty()) {
+      le->setText(QDir::toNativeSeparators(dir));
+      le->setFocus();
+    }
+  });
+
+  QObject::connect(le, &QLineEdit::textChanged, [=](const QString & text) {
+    *path = text;
+    if (QFile::exists(text))
+      le->setStyleSheet("");
+    else
+      le->setStyleSheet("QLineEdit {color: red;}");
+  });
+  le->setText(QDir::toNativeSeparators(*path));
+
+  return fsw;
+}
+
 void MainWindow::sdsync()
 {
-  QString sdPath = g.profile[g.id()].sdPath();
-  if (sdPath.isEmpty()) {
-    QMessageBox::warning(this, QObject::tr("Synchronization error"), QObject::tr("No SD directory configured!"));
+  const QString dlgTtl = tr("Synchronize SD");
+  const QIcon dlgIcn = CompanionIcon("sdsync.png");
+  const QString srcArw = CPN_STR_SW_INDICATOR_UP % " ";
+  const QString dstArw = CPN_STR_SW_INDICATOR_DN % " ";
+  QStringList errorMsgs;
+
+  // remember user-selectable options for duration of session
+  static QString sourcePath;
+  static QString destPath;
+  static int syncDirection = SyncProcess::SYNC_A2B_B2A;
+  static int compareType = SyncProcess::OVERWR_NEWER_IF_DIFF;
+  static int maxFileSize = 2 * 1024 * 1024;  // Bytes
+  static bool dryRun = false;
+
+  if (sourcePath.isEmpty())
+    sourcePath = g.profile[g.id()].sdPath();
+  if (destPath.isEmpty())
+    destPath = findMassstoragePath("SOUNDS").replace(QRegExp("[/\\\\]?SOUNDS"), "");
+
+  if (sourcePath.isEmpty())
+    errorMsgs << tr("No local SD structure path configured!");
+  if (destPath.isEmpty())
+    errorMsgs << tr("No Radio or SD card detected!");
+
+  QDialog dlg(this);
+  dlg.setWindowTitle(dlgTtl % tr(" :: Options"));
+  dlg.setWindowIcon(dlgIcn);
+  dlg.setSizeGripEnabled(true);
+  dlg.setWindowFlags(dlg.windowFlags() & ~Qt::WindowContextHelpButtonHint);
+
+  QLabel * lblSrc = new QLabel(tr("Local Folder:"), &dlg);
+  QWidget * wdgSrc = folderSelectorWidget(&sourcePath, lblSrc->text(), &dlg);
+
+  QLabel * lblDst = new QLabel(tr("Radio Folder:"), &dlg);
+  QWidget * wdgDst = folderSelectorWidget(&destPath, lblDst->text(), &dlg);
+
+  QLabel * lbDir = new QLabel(tr("Sync. Direction:"), &dlg);
+  QComboBox * syncDir = new QComboBox(&dlg);
+  syncDir->addItem(tr("%1%2 Both directions, to radio folder first").arg(dstArw, srcArw), SyncProcess::SYNC_A2B_B2A);
+  syncDir->addItem(tr("%1%2 Both directions, to local folder first").arg(srcArw, dstArw), SyncProcess::SYNC_B2A_A2B);
+  syncDir->addItem(tr(" %1  Only from local folder to radio folder").arg(dstArw), SyncProcess::SYNC_A2B);
+  syncDir->addItem(tr(" %1  Only from radio folder to local folder").arg(srcArw), SyncProcess::SYNC_B2A);
+  syncDir->setCurrentIndex(-1);  // we set the default option later
+
+  QLabel * lbMode = new QLabel(tr("Existing Files:"), &dlg);
+  QComboBox * copyMode = new QComboBox(&dlg);
+  copyMode->setToolTip(tr("How to handle overwriting files which already exist in the destination folder."));
+  copyMode->addItem(tr("Copy only if newer and different (compare contents)"), SyncProcess::OVERWR_NEWER_IF_DIFF);
+  copyMode->addItem(tr("Copy only if newer (do not compare contents)"), SyncProcess::OVERWR_NEWER_ALWAYS);
+  copyMode->addItem(tr("Copy only if different (ignore file time stamps)"), SyncProcess::OVERWR_IF_DIFF);
+  copyMode->addItem(tr("Always copy (force overwite existing files)"), SyncProcess::OVERWR_ALWAYS);
+
+  QLabel * lbSize = new QLabel(tr("Max. File Size:"), &dlg);
+  QSpinBox * maxSize = new QSpinBox(&dlg);
+  maxSize->setRange(0, 100 * 1024);
+  maxSize->setAccelerated(true);
+  maxSize->setSpecialValueText(tr("Any size"));
+  maxSize->setToolTip(tr("Skip files larger than this size. Enter zero for unlimited."));
+#if (QT_VERSION >= QT_VERSION_CHECK(5, 3, 0))
+  maxSize->setGroupSeparatorShown(true);
+#endif
+
+  QCheckBox * testRun = new QCheckBox(tr("Test-run only"), &dlg);
+  testRun->setToolTip(tr("Run as normal but do not actually copy anything. Useful for verifying results before real sync."));
+  connect(testRun, &QCheckBox::toggled, [=](bool on) { dryRun = on; });
+
+  // layout to hold size spinbox and checkbox option(s)
+  QHBoxLayout * hlay1 = new QHBoxLayout();
+  hlay1->addWidget(maxSize, 1);
+  hlay1->addWidget(testRun);
+
+  // dialog OK/Cancel buttons
+  QDialogButtonBox * bb = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel, &dlg);
+
+  // Create main layout and add everything
+  QGridLayout * dlgL = new QGridLayout(&dlg);
+  dlgL->setSizeConstraint(QLayout::SetFixedSize);
+  int row = 0;
+  if (errorMsgs.size()) {
+    QLabel * lblWarn = new QLabel(QString(errorMsgs.join('\n')), &dlg);
+    lblWarn->setStyleSheet("QLabel { color: red; }");
+    dlgL->addWidget(lblWarn, row++, 0, 1, 2);
+  }
+  dlgL->addWidget(lblSrc, row, 0);
+  dlgL->addWidget(wdgSrc, row++, 1);
+  dlgL->addWidget(lblDst, row, 0);
+  dlgL->addWidget(wdgDst, row++, 1);
+  dlgL->addWidget(lbDir, row, 0);
+  dlgL->addWidget(syncDir, row++, 1);
+  dlgL->addWidget(lbMode, row, 0);
+  dlgL->addWidget(copyMode, row++, 1);
+  dlgL->addWidget(lbSize, row, 0);
+  dlgL->addLayout(hlay1, row++, 1);
+  dlgL->addWidget(bb, row++, 0, 1, 2);
+  dlgL->setRowStretch(row, 1);
+
+  connect(copyMode, static_cast<void(QComboBox::*)(int)>(&QComboBox::currentIndexChanged), [=](int) {
+    compareType = copyMode->currentData().toInt();
+  });
+
+  // function to dis/enable the OVERWR_ALWAYS option depending on sync direction
+  connect(syncDir, static_cast<void(QComboBox::*)(int)>(&QComboBox::currentIndexChanged), [=](int) {
+    int dir = syncDir->currentData().toInt();
+    int idx = copyMode->findData(SyncProcess::OVERWR_ALWAYS);
+    int flg = (dir == SyncProcess::SYNC_A2B || dir == SyncProcess::SYNC_B2A) ? 33 : 0;
+    if (!flg && idx == copyMode->currentIndex())
+      copyMode->setCurrentIndex(copyMode->findData(SyncProcess::OVERWR_NEWER_IF_DIFF));
+    copyMode->setItemData(idx, flg, Qt::UserRole - 1);
+    syncDirection = dir;
+  });
+
+  // function to set magnitude of file size spinbox, KB or MB
+  connect(maxSize, static_cast<void(QSpinBox::*)(int)>(&QSpinBox::valueChanged), [=](int value) {
+    int multi = maxSize->property("multi").isValid() ? maxSize->property("multi").toInt() : 0;
+    maxSize->blockSignals(true);
+    if (value >= 10 * 1024 && multi != 1024 * 1024) {
+      // KB -> MB
+      multi = 1024 * 1024;
+      maxSize->setValue(value / 1024);
+      maxSize->setMaximum(100);
+      maxSize->setSingleStep(1);
+      maxSize->setSuffix(tr(" MB"));
+    }
+    else if ((value < 10 && multi != 1024) || !multi) {
+      // MB -> KB
+      if (multi)
+        value *= 1024;
+      multi = 1024;
+      if (value == 9 * 1024)
+        value += 1024 - 32;  // avoid large jump when stepping from 10MB to 10,208KB
+      maxSize->setMaximum(100 * 1024);
+      maxSize->setValue(value);
+      maxSize->setSingleStep(32);
+      maxSize->setSuffix(tr(" KB"));
+    }
+    maxSize->setProperty("multi", multi);
+    maxSize->blockSignals(false);
+    maxFileSize = value * multi;
+  });
+
+  copyMode->setCurrentIndex(copyMode->findData(compareType));
+  syncDir->setCurrentIndex(syncDir->findData(syncDirection));
+  maxSize->setValue(maxFileSize / 1024);
+  testRun->setChecked(dryRun);
+
+  connect(bb, &QDialogButtonBox::accepted, &dlg, &QDialog::accept);
+  connect(bb, &QDialogButtonBox::rejected, &dlg, &QDialog::reject);
+
+  // to restart dialog on error/etc
+  openDialog:
+
+  // show the modal options dialog
+  if (dlg.exec() == QDialog::Rejected)
     return;
+
+  // validate
+  errorMsgs.clear();
+  if (sourcePath == destPath)
+    errorMsgs << tr("Source and destination folders are the same!");
+  if (sourcePath.isEmpty() || !QFile::exists(sourcePath))
+    errorMsgs << tr("Source folder not found: %1").arg(sourcePath);
+  if (destPath.isEmpty() || !QFile::exists(destPath))
+    errorMsgs << tr("Destination folder not found: %1").arg(destPath);
+
+  if (!errorMsgs.isEmpty()) {
+    QMessageBox::warning(this, dlgTtl % tr(" :: Error"), errorMsgs.join('\n'));
+    goto openDialog;
   }
-  QString massstoragePath = findMassstoragePath("SOUNDS");
-  if (massstoragePath.isEmpty()) {
-    QMessageBox::warning(this, QObject::tr("Synchronization error"), QObject::tr("No Radio connected!"));
-    return;
-  }
-  massstoragePath = massstoragePath.left(massstoragePath.length() - 7);
-  ProgressDialog progressDialog(this, tr("Synchronize SD"), CompanionIcon("sdsync.png"));
-  SyncProcess syncProcess(massstoragePath, g.profile[g.id()].sdPath(), progressDialog.progress());
-  if (!syncProcess.run()) {
-    progressDialog.exec();
-  }
+
+  // set up the progress dialog and the sync process worker
+  ProgressDialog * progressDlg = new ProgressDialog(this, dlgTtl % tr(" :: Progress"), dlgIcn);
+  progressDlg->setAttribute(Qt::WA_DeleteOnClose, true);
+  ProgressWidget * progWidget = progressDlg->progress();
+  SyncProcess * syncProcess = new SyncProcess(sourcePath, destPath, syncDirection, compareType, maxFileSize, dryRun);
+
+  // move sync process to separate thread, we only use signals/slots from here on...
+  QThread * syncThread = new QThread(this);
+  syncProcess->moveToThread(syncThread);
+
+  // ...and quite a few of them!
+  connect(this,        &MainWindow::startSync,         syncProcess, &SyncProcess::run);
+  connect(syncThread,  &QThread::finished,             syncProcess, &SyncProcess::deleteLater);
+  connect(syncProcess, &SyncProcess::finished,         syncThread,  &QThread::quit);
+  connect(syncProcess, &SyncProcess::destroyed,        syncThread,  &QThread::quit);
+  connect(syncProcess, &SyncProcess::destroyed,        syncThread,  &QThread::deleteLater);
+  connect(syncProcess, &SyncProcess::fileCountChanged, progWidget,  &ProgressWidget::setMaximum);
+  connect(syncProcess, &SyncProcess::progressStep,     progWidget,  &ProgressWidget::setValue);
+  connect(syncProcess, &SyncProcess::progressMessage,  progWidget,  &ProgressWidget::addMessage);
+  connect(syncProcess, &SyncProcess::statusMessage,    progWidget,  &ProgressWidget::setInfo);
+  connect(syncProcess, &SyncProcess::started,          progressDlg, &ProgressDialog::setProcessStarted);
+  connect(syncProcess, &SyncProcess::finished,         progressDlg, &ProgressDialog::setProcessStopped);
+  connect(syncProcess, &SyncProcess::finished,         [=]()        { QApplication::alert(this); });
+  connect(progressDlg, &ProgressDialog::rejected,      syncProcess, &SyncProcess::stop);
+  connect(progressDlg, &ProgressDialog::rejected,      syncProcess, &SyncProcess::deleteLater);
+
+  // go (finally)
+  syncThread->start();
+  emit startSync();
 }
 
 void MainWindow::changelog()

companion/src/mainwindow.h

@@ -54,6 +54,7 @@ class MainWindow : public QMainWindow
 
   signals:
     void firmwareChanged();
+    void startSync();
 
   protected:
     QString getCompanionUpdateBaseUrl();

companion/src/process_sync.cpp

@@ -19,133 +19,256 @@
  */
 
 #include "process_sync.h"
-#include "progresswidget.h"
-#include <QDirIterator>
+
+#include <QApplication>
+#include <QCryptographicHash>
 #include <QDateTime>
-#include <QMessageBox>
-#include <QTextStream>
+#include <QMutexLocker>
+#include <QDirIterator>
 #include <QDebug>
-#include <QEventLoop>
-#include <QTimer>
 
-SyncProcess::SyncProcess(const QString & folder1, const QString & folder2, ProgressWidget * progress):
-  folder1(folder1),
-  folder2(folder2),
-  progress(progress),
-  index(0),
-  count(0),
-  closed(false)
+#if (QT_VERSION < QT_VERSION_CHECK(5, 5, 0))
+#define QtInfoMsg    QtMsgType(4)
+#endif
+
+#define PRINT_INFO(str)       emit progressMessage((str), QtInfoMsg)
+#define PRINT_CREATE(str)     emit progressMessage((str), QtInfoMsg)
+#define PRINT_REPLACE(str)    emit progressMessage((str), QtWarningMsg)
+#define PRINT_DELETE(str)     emit progressMessage((str), QtCriticalMsg)
+#define PRINT_ERROR(str)      emit progressMessage((str), QtFatalMsg)
+//#define PRINT_SKIP(str)     emit progressMessage((str), QtDebugMsg)  // mostly useless noise (maybe make an option later)
+#define PRINT_SKIP(str)
+#define PRINT_SEP()           PRINT_INFO(QString(80, '='))
+
+#define SYNC_MAX_ERRORS         50  // give up after this many errors per destination
+
+SyncProcess::SyncProcess(const QString & folderA, const QString & folderB, const int & syncDirection, const int & compareType, const qint64 & maxFileSize, const bool dryRun):
+  folder1(folderA),
+  folder2(folderB),
+  direction((SyncDirection)syncDirection),
+  ctype((SyncCompareType)compareType),
+  maxFileSize(qMax<qint64>(0, maxFileSize)),
+  dryRun(dryRun),
+  stopping(false)
 {
-  connect(progress, SIGNAL(stopped()),this, SLOT(onClosed()));
+  if (direction == SYNC_B2A_A2B) {
+    folder1 = folderB;
+    folder2 = folderA;
+    direction = SYNC_A2B_B2A;
+  }
+
+  if (ctype == OVERWR_ALWAYS && direction == SYNC_A2B_B2A)
+    ctype = OVERWR_IF_DIFF;
+
+  dirFilters = QDir::Filters(QDir::AllDirs | QDir::Files | QDir::NoDotAndDotDot);
+
+  reportTemplate = tr("New: <b>%1</b>; Updated: <b>%2</b>; Skipped: <b>%3</b>; Errors: <font color=%5><b>%4</b></font>;");
+  if (dryRun)
+    testRunStr = tr("[TEST RUN] ");
 }
 
-void SyncProcess::onClosed()
+void SyncProcess::stop()
 {
-  closed = true;
+  QMutexLocker locker(&stopReqMutex);
+  stopping = true;
 }
 
-bool SyncProcess::run()
+bool SyncProcess::isStopRequsted()
 {
-  if (!QFile::exists(folder1)) {
-    QMessageBox::warning(NULL, QObject::tr("Synchronization error"), QObject::tr("The directory '%1' doesn't exist!").arg(folder1));
-    return true;
+  QMutexLocker locker(&stopReqMutex);
+  return stopping;
+}
+
+void SyncProcess::run()
+{
+  count = index = created = updated = skipped = errored = 0;
+
+  emit started();
+  emit progressStep(index);
+  emit statusMessage(tr("Gathering file information..."));
+
+  if (direction == SYNC_A2B_B2A || direction == SYNC_A2B)
+    count += getFilesCount(folder1);
+
+  if (isStopRequsted()) {
+    finish();
+    return;
   }
 
-  if (!QFile::exists(folder2)) {
-    QMessageBox::warning(NULL, QObject::tr("Synchronization error"), QObject::tr("The directory '%1' doesn't exist!").arg(folder2));
-    return true;
+  if (direction == SYNC_A2B_B2A || direction == SYNC_B2A)
+    count += getFilesCount(folder2);
+
+  if (isStopRequsted()) {
+    finish();
+    return;
   }
 
-  count = getFilesCount(folder1) + getFilesCount(folder2);
-  progress->setMaximum(count);
+  emit fileCountChanged(count);
 
-  QStringList errors = updateDir(folder1, folder2) + updateDir(folder2, folder1);
-  if (errors.count() > 0) {
-    QMessageBox::warning(NULL, QObject::tr("Synchronization error"), errors.join("\n"));
+  if (!count) {
+    QString nf = tr("Synchronization failed, nothing found to copy.");
+    emit statusMessage(nf);
+    PRINT_ERROR(nf);
+    emit finished();
+    return;
   }
 
-  // don't close the window unless the user wanted
-  return closed;
+  if (direction == SYNC_A2B_B2A || direction == SYNC_A2B)
+    updateDir(folder1, folder2);
+
+  if (isStopRequsted()) {
+    finish();
+    return;
+  }
+
+  if (direction == SYNC_A2B_B2A || direction == SYNC_B2A)
+    updateDir(folder2, folder1);
+
+  finish();
+}
+
+void SyncProcess::finish()
+{
+  QString endStr = testRunStr % tr("Synchronization finished. ") % reportTemplate;
+  emit statusMessage(endStr.arg(created).arg(updated).arg(skipped).arg(errored).arg(errored ? "red" : "black"));
+  emit finished();
 }
 
 int SyncProcess::getFilesCount(const QString & directory)
 {
+  if (!QFile::exists(directory))
+    return 0;
+
   int result = 0;
-  QDirIterator it(directory, QDirIterator::Subdirectories);
-  while (it.hasNext()) {
+  QDirIterator it(directory, dirFilters, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
+  while (it.hasNext() && !isStopRequsted()) {
     it.next();
     result++;
+    QApplication::processEvents();
   }
   return result;
 }
 
-QStringList SyncProcess::updateDir(const QString & source, const QString & destination)
+void SyncProcess::updateDir(const QString & source, const QString & destination)
 {
-  QDirIterator it(source, QDirIterator::Subdirectories);
-  while (!closed && it.hasNext()) {
-    QEventLoop loop;
-    QTimer::singleShot(10, &loop, SLOT(quit()));
-    loop.exec();
-    index++;
-    progress->setInfo(tr("%1/%2 files").arg(index).arg(count));
-    progress->setValue(index);
-    QString result = updateEntry(it.next(), source, destination);
-    if (!result.isEmpty()) {
-      errors << result;
+  int counts[4] = { created, updated, skipped, errored };
+  QString statusStr =  testRunStr % tr("Synchronizing %1 -&gt; %2: %3").arg(source, destination, "<b>%1</b>|<b>%2</b> (%3)");
+
+  PRINT_INFO(testRunStr % tr("Starting synchronization: %1 -&gt; %2<br>").arg(source, destination));
+
+  QDirIterator it(source, dirFilters, QDirIterator::Subdirectories | QDirIterator::FollowSymlinks);
+  while (it.hasNext() && !isStopRequsted()) {
+    it.next();
+    ++index;
+    emit statusMessage(statusStr.arg(index).arg(count).arg(reportTemplate.arg(created).arg(updated).arg(skipped).arg(errored).arg(errored ? "red" : "black")));
+    emit progressStep(index);
+    if (maxFileSize && it.fileInfo().isFile() && it.fileInfo().size() > maxFileSize) {
+      PRINT_SKIP(tr("Skipping large file: %1 (%2KB)").arg(it.fileName()).arg(int(it.fileInfo().size() / 1024)));
+      ++skipped;
     }
+    else {
+      updateEntry(it.filePath(), source, destination);
+      if (errored - counts[3] > SYNC_MAX_ERRORS) {
+        PRINT_ERROR(tr("<br><b>Too many errors, giving up.<b>"));
+        break;
+      }
+    }
+    QApplication::processEvents();
   }
-  return errors;
+
+  QString endStr = "<br>" % testRunStr % tr("Finished synchronizing %1 -&gt; %2 :<br>&nbsp;&nbsp;&nbsp;&nbsp; %3").arg(source, destination, reportTemplate);
+  PRINT_INFO(endStr.arg(created-counts[0]).arg(updated-counts[1]).arg(skipped-counts[2]).arg(errored-counts[3]).arg(errored-counts[3] ? "red" : "black"));
+  PRINT_SEP();
 }
 
-QString SyncProcess::updateEntry(const QString & path, const QDir & source, const QDir & destination)
+bool SyncProcess::updateEntry(const QString & entry, const QDir & source, const QDir & destination)
 {
-  QFileInfo sourceInfo(path);
-  QString relativePath = source.relativeFilePath(path);
-  QString destinationPath = destination.absoluteFilePath(relativePath);
-  QFileInfo destinationInfo(destinationPath);
+  QString srcPath = source.toNativeSeparators(source.absoluteFilePath(entry));
+  QString relPath = source.relativeFilePath(entry);
+  QString destPath = destination.toNativeSeparators(destination.absoluteFilePath(relPath));
+  QFileInfo sourceInfo(srcPath);
+  QFileInfo destInfo(destPath);
+
   if (sourceInfo.isDir()) {
-    if (!destinationInfo.exists()) {
-      progress->addText(tr("Create directory %1\n").arg(destinationPath));
-      if (!destination.mkdir(relativePath)) {
-        return QObject::tr("Create '%1' failed").arg(destinationPath);
+    if (!destInfo.exists()) {
+      PRINT_CREATE(tr("Creating directory: %1").arg(destPath));
+      if (!dryRun && !destination.mkpath(destPath)) {
+        PRINT_ERROR(tr("Could not create directory: %1").arg(destPath));
+        ++errored;
+        return false;
       }
+      ++created;
     }
+    else {
+      PRINT_SKIP(tr("Destination directory exists: %1").arg(destPath));
+      ++skipped;
+    }
+    return true;
   }
-  else {
-    if (!destinationInfo.exists()) {
-      // qDebug() << "Copy" << path << "to" << destinationPath;
-      progress->addText(tr("Copy %1 to %2").arg(path).arg(destinationPath) + "\n");
-      if (!QFile::copy(path, destinationPath)) {
-        return QObject::tr("Copy '%1' to '%2' failed").arg(path).arg(destinationPath);
-      }
-    }
-    else if (sourceInfo.lastModified() > destinationInfo.lastModified()) {
-      // retrieve source contents
-      QFile sourceFile(path);
-      if (!sourceFile.open(QFile::ReadOnly)) {
-        return QObject::tr("Open '%1' failed").arg(path);
-      }
-      QString sourceContents = sourceFile.readAll();
-      sourceFile.close();
-      // try to retrieve destination contents
-      QFile destinationFile(destinationPath);
-      if (destinationFile.open(QFile::ReadOnly)) {
-        QString destinationContents = destinationFile.readAll();
-        destinationFile.close();
-        if (sourceContents == destinationContents) {
-          // qDebug() << "Skip" << path;
-          return QString();
-        }
-      }
-      if (!destinationFile.open(QFile::WriteOnly)) {
-        return QObject::tr("Write '%1' failed").arg(destinationPath);
-      }
-      progress->addText(tr("Write %1").arg(destinationPath) + "\n");
-      // qDebug() << "Write" << destinationPath;
-      QTextStream destinationStream(&destinationFile);
-      destinationStream << sourceContents;
-      destinationFile.close();
+
+  QFile sourceFile(srcPath);
+  QFile destinationFile(destPath);
+  bool destExists = destInfo.exists();
+  bool checkDate = (ctype == OVERWR_NEWER_IF_DIFF || ctype == OVERWR_NEWER_ALWAYS);
+  bool checkContent = (ctype == OVERWR_NEWER_IF_DIFF || ctype == OVERWR_IF_DIFF);
+  bool existed = false;
+
+  if (destExists && checkDate) {
+    if (sourceInfo.lastModified() <= destInfo.lastModified()) {
+      PRINT_SKIP(tr("Skipping older file: %1").arg(srcPath));
+      ++skipped;
+      return true;
     }
+    checkDate = false;
   }
-  return QString();
+
+  if (destExists && checkContent) {
+    if (!sourceFile.open(QFile::ReadOnly)) {
+      PRINT_ERROR(tr("Could not open source file '%1': %2").arg(srcPath, sourceFile.errorString()));
+      ++errored;
+      return false;
+    }
+    if (!destinationFile.open(QFile::ReadOnly)) {
+      PRINT_ERROR(tr("Could not open destination file '%1': %2").arg(destPath, destinationFile.errorString()));
+      ++errored;
+      return false;
+    }
+
+    bool skip =  QCryptographicHash::hash(sourceFile.readAll(), QCryptographicHash::Md5) == QCryptographicHash::hash(destinationFile.readAll(), QCryptographicHash::Md5);
+    sourceFile.close();
+    destinationFile.close();
+    if (skip) {
+      PRINT_SKIP(tr("Skipping identical file: %1").arg(srcPath));
+      ++skipped;
+      return true;
+    }
+    checkContent = false;
+  }
+
+  if (!destExists || (!checkDate && !checkContent)) {
+    if (destInfo.exists()) {
+      existed = true;
+      PRINT_REPLACE(tr("Replacing destination file: %1").arg(destPath));
+      if (!dryRun && !destinationFile.remove()) {
+        PRINT_ERROR(tr("Could not delete destination file '%1': %2").arg(destPath, destinationFile.errorString()));
+        ++errored;
+        return false;
+      }
+    }
+    else {
+      PRINT_CREATE(tr("Creating destination file: %1").arg(destPath));
+    }
+    if (!dryRun && !sourceFile.copy(destPath)) {
+      PRINT_ERROR(tr("Copy failed: '%1' to '%2': %3").arg(srcPath, destPath, sourceFile.errorString()));
+      ++errored;
+      return false;
+    }
+
+    if (existed)
+      ++updated;
+    else
+      ++created;
+  }
+
+  return true;
 }

companion/src/process_sync.h

@@ -22,34 +22,72 @@
 #define _PROCESS_SYNC_H_
 
 #include <QObject>
+#include <QDir>
+#include <QMutex>
 #include <QString>
-#include <QStringList>
-
-class QDir;
-class ProgressWidget;
 
 class SyncProcess : public QObject
 {
     Q_OBJECT
 
   public:
-    SyncProcess(const QString & folder1, const QString & folder2, ProgressWidget * progress);
-    bool run();
+    enum SyncDirection {
+      SYNC_A2B_B2A,
+      SYNC_B2A_A2B,
+      SYNC_A2B,
+      SYNC_B2A
+    };
 
-  protected slots:
-    void onClosed();
+    enum SyncCompareType {
+      OVERWR_NEWER_IF_DIFF,
+      OVERWR_NEWER_ALWAYS,
+      OVERWR_IF_DIFF,
+      OVERWR_ALWAYS
+    };
+
+    SyncProcess(const QString & folderA,
+                const QString & folderB,
+                const int & syncDirection = SYNC_A2B_B2A,
+                const int & compareType = OVERWR_NEWER_IF_DIFF,
+                const qint64 & maxFileSize = 5*1024*1024,
+                const bool dryRun = false);
+
+  public slots:
+    void run();
+    void stop();
+
+  signals:
+    void started();
+    void finished();
+    void fileCountChanged(int count);
+    void progressStep(int step);
+    void progressMessage(const QString & text, const int & type);
+    void statusMessage(const QString & text);
 
   protected:
+    bool isStopRequsted();
+    void finish();
     int getFilesCount(const QString & directory);
-    QStringList updateDir(const QString & source, const QString & destination);
-    QString updateEntry(const QString & path, const QDir & source, const QDir & destination);
+    void updateDir(const QString & source, const QString & destination);
+    bool updateEntry(const QString & entry, const QDir & source, const QDir & destination);
+
     QString folder1;
     QString folder2;
-    ProgressWidget * progress;
-    QStringList errors;
+    SyncDirection direction;
+    SyncCompareType ctype;
+    qint64 maxFileSize;  // Bytes
+    QDir::Filters dirFilters;
+    QMutex stopReqMutex;
+    QString reportTemplate;
+    QString testRunStr;
     int index;
     int count;
-    bool closed;
+    int created;
+    int updated;
+    int skipped;
+    int errored;
+    bool dryRun;
+    bool stopping;
 };
 
 #endif // _PROCESS_SYNC_H_

companion/src/progressdialog.cpp

@@ -32,6 +32,7 @@ locked(false)
   ui->setupUi(this);
   setWindowTitle(title);
   setWindowIcon(icon);
+  setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint);
   if (forceOpen) {
     ui->outputProgress->forceOpen();
   }
@@ -53,6 +54,7 @@ void ProgressDialog::on_closeButton_clicked()
 {
   if (!locked) {
     ui->outputProgress->stop();
+    emit rejected();
     close();
   }
 }
@@ -77,3 +79,13 @@ bool ProgressDialog::isEmpty() const
 {
   return ui->outputProgress->isEmpty();
 }
+
+void ProgressDialog::setProcessStarted()
+{
+  ui->closeButton->setText(tr("Cancel"));
+}
+
+void ProgressDialog::setProcessStopped()
+{
+  ui->closeButton->setText(tr("Close"));
+}

companion/src/progressdialog.h

@@ -41,6 +41,10 @@ public:
   ProgressWidget * progress();
   bool isEmpty() const;
 
+public slots:
+  void setProcessStarted();
+  void setProcessStopped();
+
 private slots:
   void on_closeButton_clicked();
   void on_outputProgress_detailsToggled();

companion/src/progressdialog.ui

@@ -6,22 +6,16 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>431</width>
-    <height>115</height>
+    <width>646</width>
+    <height>211</height>
    </rect>
   </property>
   <property name="sizePolicy">
-   <sizepolicy hsizetype="MinimumExpanding" vsizetype="Minimum">
+   <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
     <horstretch>0</horstretch>
     <verstretch>0</verstretch>
    </sizepolicy>
   </property>
-  <property name="minimumSize">
-   <size>
-    <width>0</width>
-    <height>0</height>
-   </size>
-  </property>
   <property name="windowTitle">
    <string>Flash Firmware</string>
   </property>
@@ -29,28 +23,37 @@
    <iconset resource="companion.qrc">
     <normaloff>:/icon.png</normaloff>:/icon.png</iconset>
   </property>
-  <property name="modal">
-   <bool>true</bool>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout_2">
+  <layout class="QVBoxLayout" name="verticalLayout_2" stretch="1,0">
+   <property name="spacing">
+    <number>5</number>
+   </property>
+   <property name="leftMargin">
+    <number>7</number>
+   </property>
+   <property name="topMargin">
+    <number>5</number>
+   </property>
+   <property name="rightMargin">
+    <number>7</number>
+   </property>
+   <property name="bottomMargin">
+    <number>4</number>
+   </property>
    <item>
-    <widget class="ProgressWidget" name="outputProgress" native="true"/>
-   </item>
-   <item>
-    <spacer name="spacer2">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
+    <widget class="ProgressWidget" name="outputProgress" native="true">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Expanding" vsizetype="Minimum">
+       <horstretch>0</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
      </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>0</height>
-      </size>
-     </property>
-    </spacer>
+    </widget>
    </item>
    <item>
     <layout class="QHBoxLayout" name="horizontalLayout">
+     <property name="spacing">
+      <number>6</number>
+     </property>
      <item>
       <spacer name="spacer1">
        <property name="orientation">
@@ -59,7 +62,7 @@
        <property name="sizeHint" stdset="0">
         <size>
          <width>40</width>
-         <height>20</height>
+         <height>1</height>
         </size>
        </property>
       </spacer>

companion/src/progresswidget.cpp

@@ -34,14 +34,15 @@ ui(new Ui::ProgressWidget)
   ui->checkBox->hide();
   ui->textEdit->hide();
 
+  QFont newFont(ui->textEdit->font());
+  newFont.setFamily("Courier");
 #ifdef __APPLE__
-  QFont newFont("Courier", 13);
-  ui->textEdit->setFont(newFont);
+  newFont.setPointSize(13);
   ui->textEdit->setAttribute(Qt::WA_MacNormalSize);
 #elif defined WIN32 || !defined __GNUC__
-  QFont newFont("Courier", 9);
-  ui->textEdit->setFont(newFont);
+  newFont.setPointSize(9);
 #endif
+  ui->textEdit->setFont(newFont);
 }
 
 ProgressWidget::~ProgressWidget()
@@ -81,14 +82,10 @@ void ProgressWidget::setValue(int value)
   ui->progressBar->setValue(value);
 }
 
-void ProgressWidget::addText(const QString &text)
+void ProgressWidget::addText(const QString &text, const bool richText)
 {
   ui->checkBox->setVisible(true);
-
-  if (g.outputDisplayDetails()) {
-    ui->checkBox->setChecked(true);
-    ui->textEdit->setVisible(true);
-  }
+  ui->checkBox->setChecked(g.outputDisplayDetails());
 
   QTextCursor cursor(ui->textEdit->textCursor());
 
@@ -96,13 +93,51 @@ void ProgressWidget::addText(const QString &text)
   bool atEnd = (ui->textEdit->verticalScrollBar()->value() == ui->textEdit->verticalScrollBar()->maximum());
 
   cursor.movePosition(QTextCursor::End, QTextCursor::MoveAnchor, 1);
-  cursor.insertText(text);
+  if (richText)
+    cursor.insertHtml(text);
+  else
+    cursor.insertText(text);
 
   if (atEnd) {
     ui->textEdit->verticalScrollBar()->triggerAction(QAbstractSlider::SliderToMaximum);
   }
 }
 
+void ProgressWidget::addHtml(const QString & text)
+{
+  addText(text, true);
+}
+
+void ProgressWidget::addMessage(const QString & text, const int & type)
+{
+  QString color;
+  switch (type) {
+    case QtDebugMsg:
+      color = "dimgrey";  // not important messages, may be filtered out
+      break;
+    case QtWarningMsg:    // use warning level as emphasis
+      color = "darkblue";
+      break;
+    case QtCriticalMsg:   // use critical as a warning
+      color = "#ff7900";
+      break;
+    case QtFatalMsg:      // fatal for hard errors
+      color = "red";
+      break;
+    case QtInfoMsg:       // default plain text
+    default:
+      break;
+  }
+  if (color.isEmpty()) {
+    if (text.contains(QRegExp("<[^>]+>")))  // detect html (rouhgly)
+      addHtml(text % "<br>");
+    else
+      addText(text % "\n", false);
+  }
+  else
+    addHtml(QString("<font color=%1>").arg(color) % text % "</font><br>");
+}
+
 QString ProgressWidget::getText()
 {
   return ui->textEdit->toPlainText();

companion/src/progresswidget.h

@@ -23,6 +23,10 @@
 
 #include <QWidget>
 
+#if (QT_VERSION < QT_VERSION_CHECK(5, 5, 0))
+  #define QtInfoMsg    QtMsgType(4)
+#endif
+
 namespace Ui {
   class ProgressWidget;
 }
@@ -35,7 +39,9 @@ class ProgressWidget : public QWidget
     explicit ProgressWidget(QWidget *parent);
     ~ProgressWidget();
     void lock(bool lock);
-    void addText(const QString &text);
+    void addText(const QString &text, const bool richText = false);
+    void addHtml(const QString &text);
+    void addMessage(const QString & text, const int & type = QtInfoMsg);
     QString getText();
     void setInfo(const QString &text);
     void setMaximum(int value);

companion/src/progresswidget.ui

@@ -9,26 +9,26 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>700</width>
-    <height>244</height>
+    <width>753</width>
+    <height>474</height>
    </rect>
   </property>
   <property name="sizePolicy">
-   <sizepolicy hsizetype="Preferred" vsizetype="Minimum">
+   <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
     <horstretch>0</horstretch>
     <verstretch>0</verstretch>
    </sizepolicy>
   </property>
   <property name="minimumSize">
    <size>
-    <width>700</width>
+    <width>650</width>
     <height>0</height>
    </size>
   </property>
-  <property name="windowTitle">
-   <string>Form</string>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
+  <layout class="QVBoxLayout" name="verticalLayout" stretch="0,0,1,0">
+   <property name="spacing">
+    <number>6</number>
+   </property>
    <property name="leftMargin">
     <number>0</number>
    </property>
@@ -43,56 +43,79 @@
    </property>
    <item>
     <widget class="QLabel" name="info">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+       <horstretch>0</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
+     </property>
      <property name="text">
-      <string/>
+      <string notr="true"/>
      </property>
     </widget>
    </item>
    <item>
-    <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,0">
-     <item>
-      <widget class="QProgressBar" name="progressBar">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="minimumSize">
-        <size>
-         <width>0</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="value">
-        <number>0</number>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QCheckBox" name="checkBox">
-       <property name="text">
-        <string>Show Details</string>
-       </property>
-      </widget>
-     </item>
-    </layout>
+    <widget class="QWidget" name="widget" native="true">
+     <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,0">
+      <property name="spacing">
+       <number>6</number>
+      </property>
+      <property name="leftMargin">
+       <number>0</number>
+      </property>
+      <property name="topMargin">
+       <number>0</number>
+      </property>
+      <property name="rightMargin">
+       <number>0</number>
+      </property>
+      <property name="bottomMargin">
+       <number>0</number>
+      </property>
+      <item>
+       <widget class="QProgressBar" name="progressBar">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+          <horstretch>0</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+        <property name="minimumSize">
+         <size>
+          <width>0</width>
+          <height>0</height>
+         </size>
+        </property>
+        <property name="value">
+         <number>0</number>
+        </property>
+       </widget>
+      </item>
+      <item>
+       <widget class="QCheckBox" name="checkBox">
+        <property name="text">
+         <string>Show Details</string>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
    </item>
    <item>
-    <widget class="QPlainTextEdit" name="textEdit">
+    <widget class="QTextEdit" name="textEdit">
      <property name="enabled">
       <bool>true</bool>
      </property>
      <property name="sizePolicy">
-      <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+      <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
        <horstretch>0</horstretch>
        <verstretch>0</verstretch>
       </sizepolicy>
      </property>
      <property name="minimumSize">
       <size>
-       <width>600</width>
-       <height>100</height>
+       <width>0</width>
+       <height>350</height>
       </size>
      </property>
      <property name="font">
@@ -101,32 +124,40 @@
        <pointsize>10</pointsize>
       </font>
      </property>
-     <property name="lineWidth">
-      <number>80</number>
-     </property>
-     <property name="midLineWidth">
-      <number>80</number>
-     </property>
      <property name="verticalScrollBarPolicy">
       <enum>Qt::ScrollBarAlwaysOn</enum>
      </property>
      <property name="horizontalScrollBarPolicy">
       <enum>Qt::ScrollBarAsNeeded</enum>
      </property>
-     <property name="lineWrapMode">
-      <enum>QPlainTextEdit::NoWrap</enum>
+     <property name="undoRedoEnabled">
+      <bool>false</bool>
      </property>
-     <property name="readOnly">
-      <bool>true</bool>
-     </property>
-     <property name="plainText">
-      <string/>
+     <property name="html">
+      <string notr="true">&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
+&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
+p, li { white-space: pre-wrap; }
+&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Courier'; font-size:10pt; font-weight:400; font-style:normal;&quot;&gt;
+&lt;p style=&quot;-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;br /&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
      </property>
      <property name="textInteractionFlags">
-      <set>Qt::TextSelectableByMouse</set>
+      <set>Qt::LinksAccessibleByKeyboard|Qt::LinksAccessibleByMouse|Qt::TextBrowserInteraction|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse</set>
      </property>
     </widget>
    </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>0</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
   </layout>
  </widget>
  <resources/>