Trying to improve synchronization performance with database batched operations

diff --git a/src/eu/alefzero/owncloud/datamodel/DataStorageManager.java b/src/eu/alefzero/owncloud/datamodel/DataStorageManager.java
index e4c385f..9f1d89b 100644 (file)
--- a/src/eu/alefzero/owncloud/datamodel/DataStorageManager.java
+++ b/src/eu/alefzero/owncloud/datamodel/DataStorageManager.java
@@ -18,6 +18,7 @@
 
 package eu.alefzero.owncloud.datamodel;
 
 
 package eu.alefzero.owncloud.datamodel;
 

+import java.util.List;

 import java.util.Vector;
 
 public interface DataStorageManager {
 import java.util.Vector;
 
 public interface DataStorageManager {


@@ -32,6 +33,8 @@ public interface DataStorageManager {
 
     public boolean saveFile(OCFile file);
 
 
     public boolean saveFile(OCFile file);
 

+    public void saveFiles(List<OCFile> files);
+

     public Vector<OCFile> getDirectoryContent(OCFile f);
     
     public void removeFile(OCFile file);
     public Vector<OCFile> getDirectoryContent(OCFile f);
     
     public void removeFile(OCFile file);

diff --git a/src/eu/alefzero/owncloud/datamodel/FileDataStorageManager.java b/src/eu/alefzero/owncloud/datamodel/FileDataStorageManager.java
index 1feb424..e1b62ef 100644 (file)
--- a/src/eu/alefzero/owncloud/datamodel/FileDataStorageManager.java
+++ b/src/eu/alefzero/owncloud/datamodel/FileDataStorageManager.java
@@ -19,14 +19,21 @@
 package eu.alefzero.owncloud.datamodel;
 
 import java.io.File;
 package eu.alefzero.owncloud.datamodel;
 
 import java.io.File;

+import java.util.ArrayList;

 import java.util.Collections;
 import java.util.Collections;

+import java.util.Iterator;
+import java.util.List;

 import java.util.Vector;
 
 import java.util.Vector;
 

+import eu.alefzero.owncloud.db.ProviderMeta;

 import eu.alefzero.owncloud.db.ProviderMeta.ProviderTableMeta;
 import android.accounts.Account;
 import android.content.ContentProviderClient;
 import eu.alefzero.owncloud.db.ProviderMeta.ProviderTableMeta;
 import android.accounts.Account;
 import android.content.ContentProviderClient;

+import android.content.ContentProviderOperation;
+import android.content.ContentProviderResult;

 import android.content.ContentResolver;
 import android.content.ContentValues;
 import android.content.ContentResolver;
 import android.content.ContentValues;

+import android.content.OperationApplicationException;

 import android.database.Cursor;
 import android.net.Uri;
 import android.os.Environment;
 import android.database.Cursor;
 import android.net.Uri;
 import android.os.Environment;


@@ -155,6 +162,86 @@ public class FileDataStorageManager implements DataStorageManager {
         return overriden;
     }
 
         return overriden;
     }
 

+
+    @Override
+    public void saveFiles(List<OCFile> files) {
+        
+        Iterator<OCFile> filesIt = files.iterator();
+        ArrayList<ContentProviderOperation> operations = new ArrayList<ContentProviderOperation>(files.size());
+        OCFile file = null;
+
+        // prepare operations to perform
+        while (filesIt.hasNext()) {
+            file = filesIt.next();
+            ContentValues cv = new ContentValues();
+            cv.put(ProviderTableMeta.FILE_MODIFIED, file.getModificationTimestamp());
+            cv.put(ProviderTableMeta.FILE_CREATION, file.getCreationTimestamp());
+            cv.put(ProviderTableMeta.FILE_CONTENT_LENGTH, file.getFileLength());
+            cv.put(ProviderTableMeta.FILE_CONTENT_TYPE, file.getMimetype());
+            cv.put(ProviderTableMeta.FILE_NAME, file.getFileName());
+            if (file.getParentId() != 0)
+                cv.put(ProviderTableMeta.FILE_PARENT, file.getParentId());
+            cv.put(ProviderTableMeta.FILE_PATH, file.getRemotePath());
+            if (!file.isDirectory())
+                cv.put(ProviderTableMeta.FILE_STORAGE_PATH, file.getStoragePath());
+            cv.put(ProviderTableMeta.FILE_ACCOUNT_OWNER, mAccount.name);
+            cv.put(ProviderTableMeta.FILE_LAST_SYNC_DATE, file.getLastSyncDate());
+            cv.put(ProviderTableMeta.FILE_KEEP_IN_SYNC, file.keepInSync() ? 1 : 0);
+
+            if (fileExists(file.getRemotePath())) {
+                OCFile tmpfile = getFileByPath(file.getRemotePath());
+                file.setStoragePath(tmpfile.getStoragePath());
+                if (!file.isDirectory());
+                    cv.put(ProviderTableMeta.FILE_STORAGE_PATH, file.getStoragePath());
+                file.setFileId(tmpfile.getFileId());
+
+                operations.add(ContentProviderOperation.newUpdate(ProviderTableMeta.CONTENT_URI).
+                        withValues(cv).
+                        withSelection(  ProviderTableMeta._ID + "=?", 
+                                        new String[] { String.valueOf(file.getFileId()) })
+                        .build());
+                
+            } else {
+                operations.add(ContentProviderOperation.newInsert(ProviderTableMeta.CONTENT_URI).withValues(cv).build());
+            }
+        }
+        
+        // apply operations in batch
+        ContentProviderResult[] results = null;
+        try {
+            if (getContentResolver() != null) {
+                results = getContentResolver().applyBatch(ProviderMeta.AUTHORITY_FILES, operations);
+            
+            } else {
+                results = getContentProvider().applyBatch(operations);
+            }
+            
+        } catch (OperationApplicationException e) {
+            Log.e(TAG, "Fail to update/insert list of files to database " + e.getMessage());
+            
+        } catch (RemoteException e) {
+            Log.e(TAG, "Fail to update/insert list of files to database " + e.getMessage());
+        }
+        
+        // update new id in file objects for insertions
+        if (results != null) {
+            long newId;
+            for (int i=0; i<results.length; i++) {
+                if (results[i].uri != null) {
+                    newId = Long.parseLong(results[i].uri.getPathSegments().get(1));
+                    files.get(i).setFileId(newId);
+                    //Log.v(TAG, "Found and added id in insertion for " + files.get(i).getRemotePath());
+                }
+            }
+        }
+
+        for (OCFile aFile : files) {
+            if (aFile.isDirectory() && aFile.needsUpdatingWhileSaving())
+                saveFiles(getDirectoryContent(aFile));
+        }
+
+    }
+    

     public void setAccount(Account account) {
         mAccount = account;
     }
     public void setAccount(Account account) {
         mAccount = account;
     }

diff --git a/src/eu/alefzero/owncloud/providers/FileContentProvider.java b/src/eu/alefzero/owncloud/providers/FileContentProvider.java
index a354be0..7ed73a9 100644 (file)
--- a/src/eu/alefzero/owncloud/providers/FileContentProvider.java
+++ b/src/eu/alefzero/owncloud/providers/FileContentProvider.java
@@ -123,7 +123,9 @@ public class FileContentProvider extends ContentProvider {
 \r
     @Override\r
     public Uri insert(Uri uri, ContentValues values) {\r
 \r
     @Override\r
     public Uri insert(Uri uri, ContentValues values) {\r

-        if (mUriMatcher.match(uri) != SINGLE_FILE) {\r
+        if (mUriMatcher.match(uri) != SINGLE_FILE &&\r
+            mUriMatcher.match(uri) != ROOT_DIRECTORY) {\r
+            \r

             throw new IllegalArgumentException("Unknown uri id: " + uri);\r
         }\r
 \r
             throw new IllegalArgumentException("Unknown uri id: " + uri);\r
         }\r
 \r

diff --git a/src/eu/alefzero/owncloud/syncadapter/FileSyncAdapter.java b/src/eu/alefzero/owncloud/syncadapter/FileSyncAdapter.java
index 1dcc745..ef7a3d5 100644 (file)
--- a/src/eu/alefzero/owncloud/syncadapter/FileSyncAdapter.java
+++ b/src/eu/alefzero/owncloud/syncadapter/FileSyncAdapter.java
@@ -20,6 +20,7 @@ package eu.alefzero.owncloud.syncadapter;
 \r
 import java.io.IOException;\r
 import java.io.ObjectInputStream.GetField;\r
 \r
 import java.io.IOException;\r
 import java.io.ObjectInputStream.GetField;\r

+import java.util.List;\r

 import java.util.Vector;\r
 \r
 import org.apache.jackrabbit.webdav.DavException;\r
 import java.util.Vector;\r
 \r
 import org.apache.jackrabbit.webdav.DavException;\r


@@ -50,6 +51,14 @@ public class FileSyncAdapter extends AbstractOwnCloudSyncAdapter {
 \r
     private final static String TAG = "FileSyncAdapter"; \r
     \r
 \r
     private final static String TAG = "FileSyncAdapter"; \r
     \r

+    /*  Commented code for ugly performance tests\r
+    private final static int MAX_DELAYS = 100;\r
+    private static long[] mResponseDelays = new long[MAX_DELAYS]; \r
+    private static long[] mSaveDelays = new long[MAX_DELAYS];\r
+    private int mDelaysIndex = 0;\r
+    private int mDelaysCount = 0;\r
+    */\r
+    \r

     private long mCurrentSyncTime;\r
     \r
     public FileSyncAdapter(Context context, boolean autoInitialize) {\r
     private long mCurrentSyncTime;\r
     \r
     public FileSyncAdapter(Context context, boolean autoInitialize) {\r


@@ -66,6 +75,12 @@ public class FileSyncAdapter extends AbstractOwnCloudSyncAdapter {
         this.setStorageManager(new FileDataStorageManager(account,\r
                 getContentProvider()));\r
         \r
         this.setStorageManager(new FileDataStorageManager(account,\r
                 getContentProvider()));\r
         \r

+        /*  Commented code for ugly performance tests\r
+        mDelaysIndex = 0;\r
+        mDelaysCount = 0;\r
+        */\r
+        \r
+        \r

         Log.d(TAG, "syncing owncloud account " + account.name);\r
 \r
         sendStickyBroadcast(true, null);  // message to signal the start to the UI\r
         Log.d(TAG, "syncing owncloud account " + account.name);\r
 \r
         sendStickyBroadcast(true, null);  // message to signal the start to the UI\r


@@ -101,21 +116,50 @@ public class FileSyncAdapter extends AbstractOwnCloudSyncAdapter {
             Log.e(TAG, "problem while synchronizing owncloud account " + account.name, t);\r
             t.printStackTrace();\r
         }\r
             Log.e(TAG, "problem while synchronizing owncloud account " + account.name, t);\r
             t.printStackTrace();\r
         }\r

+        \r
+        /*  Commented code for ugly performance tests\r
+        long sum = 0, mean = 0, max = 0, min = Long.MAX_VALUE;\r
+        for (int i=0; i<MAX_DELAYS && i<mDelaysCount; i++) {\r
+            sum += mResponseDelays[i];\r
+            max = Math.max(max, mResponseDelays[i]);\r
+            min = Math.min(min, mResponseDelays[i]);\r
+        }\r
+        mean = sum / mDelaysCount;\r
+        Log.e(TAG, "SYNC STATS - response: mean time = " + mean + " ; max time = " + max + " ; min time = " + min);\r
+        \r
+        sum = 0; max = 0; min = Long.MAX_VALUE;\r
+        for (int i=0; i<MAX_DELAYS && i<mDelaysCount; i++) {\r
+            sum += mSaveDelays[i];\r
+            max = Math.max(max, mSaveDelays[i]);\r
+            min = Math.min(min, mSaveDelays[i]);\r
+        }\r
+        mean = sum / mDelaysCount;\r
+        Log.e(TAG, "SYNC STATS - save:     mean time = " + mean + " ; max time = " + max + " ; min time = " + min);\r
+        Log.e(TAG, "SYNC STATS - folders measured: " + mDelaysCount);\r
+        */\r
+        \r

         sendStickyBroadcast(false, null);        \r
     }\r
 \r
     private void fetchData(String uri, SyncResult syncResult, long parentId, Account account) {\r
         try {\r
         sendStickyBroadcast(false, null);        \r
     }\r
 \r
     private void fetchData(String uri, SyncResult syncResult, long parentId, Account account) {\r
         try {\r

-            Log.v(TAG, "syncing: fetching " + uri);\r
+            //Log.v(TAG, "syncing: fetching " + uri);\r

             \r
             // remote request \r
             PropFindMethod query = new PropFindMethod(uri);\r
             \r
             // remote request \r
             PropFindMethod query = new PropFindMethod(uri);\r

+            /*  Commented code for ugly performance tests\r
+            long responseDelay = System.currentTimeMillis();\r
+            */\r

             getClient().executeMethod(query);\r
             getClient().executeMethod(query);\r

+            /*  Commented code for ugly performance tests\r
+            responseDelay = System.currentTimeMillis() - responseDelay;\r
+            Log.e(TAG, "syncing: RESPONSE TIME for " + uri + " contents, " + responseDelay + "ms");\r
+            */\r

             MultiStatus resp = null;\r
             MultiStatus resp = null;\r

-            \r

             resp = query.getResponseBodyAsMultiStatus();\r
             \r
             resp = query.getResponseBodyAsMultiStatus();\r
             \r

-            // insertion of updated files\r
+            // insertion or update of files\r
+            List<OCFile> updatedFiles = new Vector<OCFile>(resp.getResponses().length - 1);\r

             for (int i = 1; i < resp.getResponses().length; ++i) {\r
                 WebdavEntry we = new WebdavEntry(resp.getResponses()[i], getUri().getPath());\r
                 OCFile file = fillOCFile(we);\r
             for (int i = 1; i < resp.getResponses().length; ++i) {\r
                 WebdavEntry we = new WebdavEntry(resp.getResponses()[i], getUri().getPath());\r
                 OCFile file = fillOCFile(we);\r


@@ -134,12 +178,21 @@ public class FileSyncAdapter extends AbstractOwnCloudSyncAdapter {
                 }\r
                 if (getStorageManager().getFileByPath(file.getRemotePath()) != null)\r
                     file.setKeepInSync(getStorageManager().getFileByPath(file.getRemotePath()).keepInSync());\r
                 }\r
                 if (getStorageManager().getFileByPath(file.getRemotePath()) != null)\r
                     file.setKeepInSync(getStorageManager().getFileByPath(file.getRemotePath()).keepInSync());\r

-                getStorageManager().saveFile(file);\r
+                //getStorageManager().saveFile(file);\r
+                updatedFiles.add(file);\r

                 if (parentId == 0)\r
                     parentId = file.getFileId();\r
             }\r
                 if (parentId == 0)\r
                     parentId = file.getFileId();\r
             }\r

+            /*  Commented code for ugly performance tests\r
+            long saveDelay = System.currentTimeMillis();\r
+            */            \r
+            getStorageManager().saveFiles(updatedFiles);    // all "at once" ; trying to get a best performance in database update\r
+            /*  Commented code for ugly performance tests\r
+            saveDelay = System.currentTimeMillis() - saveDelay;\r
+            Log.e(TAG, "syncing: SAVE TIME for " + uri + " contents, " + mSaveDelays[mDelaysIndex] + "ms");\r
+            */\r

             \r
             \r

-            // removal of old files\r
+            // removal of obsolete files\r

             Vector<OCFile> files = getStorageManager().getDirectoryContent(\r
                     getStorageManager().getFileById(parentId));\r
             OCFile file;\r
             Vector<OCFile> files = getStorageManager().getDirectoryContent(\r
                     getStorageManager().getFileById(parentId));\r
             OCFile file;\r


@@ -162,6 +215,16 @@ public class FileSyncAdapter extends AbstractOwnCloudSyncAdapter {
                     fetchData(getUri().toString() + newFile.getRemotePath(), syncResult, newFile.getFileId(), account);\r
                 }\r
             }\r
                     fetchData(getUri().toString() + newFile.getRemotePath(), syncResult, newFile.getFileId(), account);\r
                 }\r
             }\r

+            \r
+            /*  Commented code for ugly performance tests\r
+            mResponseDelays[mDelaysIndex] = responseDelay;\r
+            mSaveDelays[mDelaysIndex] = saveDelay;\r
+            mDelaysCount++;\r
+            mDelaysIndex++;\r
+            if (mDelaysIndex >= MAX_DELAYS)\r
+                mDelaysIndex = 0;\r
+             */\r
+            \r

 \r
 \r
         } catch (OperationCanceledException e) {\r
 \r
 \r
         } catch (OperationCanceledException e) {\r