/*
    ***** BEGIN LICENSE BLOCK *****
    
    Copyright © 2014 Center for History and New Media
                     George Mason University, Fairfax, Virginia, USA
                     http://zotero.org
    
    This file is part of Zotero.
    
    Zotero is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Zotero is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.
    
    You should have received a copy of the GNU Affero General Public License
    along with Zotero.  If not, see <http://www.gnu.org/licenses/>.
    
    ***** END LICENSE BLOCK *****
*/

if (!Zotero.Sync.Data) {
	Zotero.Sync.Data = {};
}

// TODO: move?
Zotero.Sync.Data.conflictDelayIntervals = [10000, 20000, 40000, 60000, 120000, 240000, 300000];

/**
 * An Engine manages sync processes for a given library
 *
 * @param {Object} options
 * @param {Zotero.Sync.APIClient} options.apiClient
 * @param {Integer} options.libraryID
 */
Zotero.Sync.Data.Engine = function (options) {
	if (options.apiClient == undefined) {
		throw new Error("options.apiClient not set");
	}
	if (options.libraryID == undefined) {
		throw new Error("options.libraryID not set");
	}
	
	this.apiClient = options.apiClient;
	this.userID = options.userID;
	this.libraryID = options.libraryID;
	this.library = Zotero.Libraries.get(options.libraryID);
	this.libraryTypeID = this.library.libraryTypeID;
	this.uploadBatchSize = 10;
	this.uploadDeletionBatchSize = 25;
	this.maxUploadTries = 5;
	
	this.failed = false;
	this.failedItems = [];
	
	// Options to pass through to processing functions
	this.optionNames = [
		'setStatus',
		'onError',
		'stopOnError',
		'background',
		'firstInSession',
		'resetMode'
	];
	this.options = {};
	this.optionNames.forEach(x => {
		// Create dummy functions if not set
		if (x == 'setStatus' || x == 'onError') {
			this[x] = options[x] || function () {};
		}
		else {
			this[x] = options[x];
		}
	});
};

Zotero.Sync.Data.Engine.prototype.DOWNLOAD_RESULT_CONTINUE = 1;
Zotero.Sync.Data.Engine.prototype.DOWNLOAD_RESULT_CHANGES_TO_UPLOAD = 2;
Zotero.Sync.Data.Engine.prototype.DOWNLOAD_RESULT_NO_CHANGES_TO_UPLOAD = 3;
Zotero.Sync.Data.Engine.prototype.DOWNLOAD_RESULT_LIBRARY_UNMODIFIED = 4;
Zotero.Sync.Data.Engine.prototype.DOWNLOAD_RESULT_RESTART = 5;

Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1;
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_NOTHING_TO_UPLOAD = 2;
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_LIBRARY_CONFLICT = 3;
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_OBJECT_CONFLICT = 4;
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_RESTART = 5;
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_CANCEL = 6;

Zotero.Sync.Data.Engine.prototype.start = async function () {
	Zotero.debug("Starting data sync for " + this.library.name);
	
	// TODO: Handle new/changed user when setting key
	if (this.library.libraryType == 'user' && !this.libraryTypeID) {
		let info = await this.apiClient.getKeyInfo();
		Zotero.debug("Got userID " + info.userID + " for API key");
		this.libraryTypeID = info.userID;
	}
	
	this._statusCheck();
	this._restoringToServer = false;
	
	// Check if we've synced this library with the current architecture yet
	var libraryVersion = this.library.libraryVersion;
	if (this.resetMode == Zotero.Sync.Runner.RESET_MODE_TO_SERVER) {
		await this._restoreToServer();
	}
	else if (!libraryVersion || libraryVersion == -1) {
		let versionResults = await this._upgradeCheck();
		if (versionResults) {
			libraryVersion = this.library.libraryVersion;
		}
		
		this._statusCheck();
		
		// Perform a full sync if necessary, passing the getVersions() results if available.
		//
		// The full-sync flag (libraryID == -1) is set at the end of a successful upgrade, so this
		// won't run for installations that have just never synced before (which also lack library
		// versions). We can't rely on last classic sync time because it's cleared after the last
		// library is upgraded.
		//
		// Version results won't be available if an upgrade happened on a previous run but the
		// full sync failed.
		if (libraryVersion == -1) {
			await this._fullSync(versionResults);
		}
	}
	
	this.downloadDelayGenerator = null;
	var autoReset = false;
	var skipDownload = false;
	
	sync:
	while (true) {
		this._statusCheck();
		
		let downloadResult, uploadResult;
		
		try {
			uploadResult = await this._startUpload();
		}
		catch (e) {
			if (e instanceof Zotero.Sync.UserCancelledException) {
				throw e;
			}
			Zotero.debug("Upload failed -- performing download", 2);
			Zotero.debug(e, 1);
			downloadResult = await this._startDownload();
			Zotero.debug("Download result is " + downloadResult, 4);
			throw e;
		}
		
		Zotero.debug("Upload result is " + uploadResult, 4);
		
		switch (uploadResult) {
		// If upload succeeded, we're done
		case this.UPLOAD_RESULT_SUCCESS:
			break sync;
		
		case this.UPLOAD_RESULT_OBJECT_CONFLICT:
			if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) {
				throw new Error("Skipping automatic client reset due to debug pref");
			}
			if (autoReset) {
				throw new Error(this.library.name + " has already been auto-reset");
			}
			Zotero.logError("Object in " + this.library.name + " is out of date -- resetting library");
			autoReset = true;
			await this._fullSync();
			break;
		
		case this.UPLOAD_RESULT_NOTHING_TO_UPLOAD:
			if (skipDownload) {
				break sync;
			}
			downloadResult = await this._startDownload();
			Zotero.debug("Download result is " + downloadResult, 4);
			if (downloadResult == this.DOWNLOAD_RESULT_CHANGES_TO_UPLOAD) {
				break;
			}
			break sync;
		
		// If conflict, start at beginning with downloads
		case this.UPLOAD_RESULT_LIBRARY_CONFLICT:
			if (!gen) {
				var gen = Zotero.Utilities.Internal.delayGenerator(
					Zotero.Sync.Data.conflictDelayIntervals, 60 * 1000
				);
			}
			// After the first upload version conflict (which is expected after remote changes),
			// start delaying to give other sync sessions time to complete
			else {
				let keepGoing = await gen.next().value;
				if (!keepGoing) {
					throw new Error("Could not sync " + this.library.name + " -- too many retries");
				}
			}
			
			downloadResult = await this._startDownload();
			Zotero.debug("Download result is " + downloadResult, 4);
			// No need to download again after the upload
			if (downloadResult !== this.DOWNLOAD_RESULT_RESTART) {
				skipDownload = true;
			}
			break;
		
		case this.UPLOAD_RESULT_RESTART:
			Zotero.debug("Restarting sync for " + this.library.name);
			break;
		
		case this.UPLOAD_RESULT_CANCEL:
			Zotero.debug("Cancelling sync for " + this.library.name);
			return;
		}
	}
	
	this.library.updateLastSyncTime();
	await this.library.saveTx({
		skipNotifier: true
	});
	
	if (this.library.libraryType == 'group') {
		try {
			await this._updateGroupItemUsers();
		}
		catch (e) {
			Zotero.logError(e);
		}
	}
	
	Zotero.debug("Done syncing " + this.library.name);
};


/**
 * Stop the sync process
 */
Zotero.Sync.Data.Engine.prototype.stop = function () {
	Zotero.debug("Stopping data sync for " + this.library.name);
	this._stopping = true;
}


/**
 * Download updated objects from API and save to DB
 *
 * @return {Promise<Integer>} - A download result code (this.DOWNLOAD_RESULT_*)
 */
Zotero.Sync.Data.Engine.prototype._startDownload = async function () {
	var localChanges = false;
	var libraryVersion = this.library.libraryVersion;
	var newLibraryVersion;
	
	loop:
	while (true) {
		this._statusCheck();
		
		// Get synced settings first, since they affect how other data is displayed
		let results = await this._downloadSettings(libraryVersion);
		if (results.result == this.DOWNLOAD_RESULT_LIBRARY_UNMODIFIED) {
			let stop = true;
			// If it's the first sync of the session or a manual sync and there are objects in the
			// sync queue, or it's a subsequent auto-sync but there are objects that it's time to try
			// again, go through all the steps even though the library version is unchanged.
			//
			// TODO: Skip the steps without queued objects.
			if (this.firstInSession || !this.background) {
				stop = !((await Zotero.Sync.Data.Local.hasObjectsInSyncQueue(this.libraryID)));
			}
			else {
				stop = !((await Zotero.Sync.Data.Local.hasObjectsToTryInSyncQueue(this.libraryID)));
			}
			if (stop) {
				Zotero.debug("Library " + this.libraryID + " hasn't been modified "
					+ "-- skipping further object downloads");
				break;
			}
		}
		newLibraryVersion = results.libraryVersion;
		
		//
		// Get other object types
		//
		for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) {
			this._statusCheck();
			
			// For items, fetch top-level items first
			//
			// The next run below will then see the same items in the non-top versions request,
			// but they'll have been downloaded already and will be skipped.
			if (objectType == 'item') {
				let result = await this._downloadUpdatedObjects(
					objectType,
					libraryVersion,
					newLibraryVersion,
					{
						top: true
					}
				);
				if (result == this.DOWNLOAD_RESULT_RESTART) {
					await this._onLibraryVersionChange();
					continue loop;
				}
			}
			
			let result = await this._downloadUpdatedObjects(
				objectType,
				libraryVersion,
				newLibraryVersion
			);
			if (result == this.DOWNLOAD_RESULT_RESTART) {
				await this._onLibraryVersionChange();
				continue loop;
			}
		}
		
		let deletionsResult = await this._downloadDeletions(libraryVersion, newLibraryVersion);
		if (deletionsResult.result == this.DOWNLOAD_RESULT_RESTART) {
			await this._onLibraryVersionChange();
			continue loop;
		}
		
		break;
	}
	
	if (newLibraryVersion) {
		// After data is downloaded, the library version is updated to match the remote version. We
		// track a library version for file syncing separately, so that even if Zotero is closed or
		// interrupted between a data sync and a file sync, we know that file syncing has to be
		// performed for any files marked for download during data sync (based on outdated mtime/md5).
		// Files may be missing remotely, though, so it's only necessary to try to download them once
		// every time there are remote storage changes, which we indicate with a 'storageDownloadNeeded'
		// flag set in syncLocal. If the storage version was already behind, though, a storage download
		// is needed, regardless of whether storage metadata was updated.
		if (this.library.storageVersion < this.library.libraryVersion) {
			this.library.storageDownloadNeeded = true;
		}
		// Update library version to match remote
		this.library.libraryVersion = newLibraryVersion;
		// Skip storage downloads if not needed
		if (!this.library.storageDownloadNeeded) {
			this.library.storageVersion = newLibraryVersion;
		}
		await this.library.saveTx();
	}
	
	return localChanges
		? this.DOWNLOAD_RESULT_CHANGES_TO_UPLOAD
		: this.DOWNLOAD_RESULT_NO_CHANGES_TO_UPLOAD;
};


/**
 * Download settings modified since the given version
 *
 * Unlike the other download methods, this method, which runs first in the main download process,
 * returns an object rather than just a download result code. It does this so it can return the
 * current library version from the API to pass to later methods, allowing them to restart the download
 * process if there was a remote change.
 *
 * @param {Integer} since - Last-known library version; get changes since this version
 * @param {Integer} [newLibraryVersion] - Newest library version seen in this sync process; if newer
 *     version is seen, restart the sync
 * @return {Object} - Object with 'result' (DOWNLOAD_RESULT_*) and 'libraryVersion'
 */
Zotero.Sync.Data.Engine.prototype._downloadSettings = async function (since, newLibraryVersion) {
	let results = await this.apiClient.getSettings(
		this.library.libraryType,
		this.libraryTypeID,
		since
	);
	// If library version hasn't changed remotely, the local library is up-to-date and we
	// can skip all remaining downloads
	if (results === false) {
		return {
			result: this.DOWNLOAD_RESULT_LIBRARY_UNMODIFIED,
			libraryVersion: since
		};
	}
	if (newLibraryVersion !== undefined && newLibraryVersion != results.libraryVersion) {
		return {
			result: this.DOWNLOAD_RESULT_RESTART,
			libraryVersion: results.libraryVersion
		};
	}
	var numObjects = Object.keys(results.settings).length;
	if (numObjects) {
		Zotero.debug(numObjects + " settings modified since last check");
		for (let setting in results.settings) {
			await Zotero.SyncedSettings.set(
				this.libraryID,
				setting,
				results.settings[setting].value,
				results.settings[setting].version,
				true
			);
		}
	}
	else {
		Zotero.debug("No settings modified remotely since last check");
	}
	return {
		result: this.DOWNLOAD_RESULT_CONTINUE,
		libraryVersion: results.libraryVersion
	};
}


/**
 * Get versions of objects updated remotely since the last sync time and kick off object downloading
 *
 * @param {String} objectType
 * @param {Integer} since - Last-known library version; get changes sinces this version
 * @param {Integer} newLibraryVersion - Last library version seen in this sync process; if newer version
 *     is seen, restart the sync
 * @param {Object} [options]
 * @return {Promise<Integer>} - A download result code (this.DOWNLOAD_RESULT_*)
 */
Zotero.Sync.Data.Engine.prototype._downloadUpdatedObjects = async function (objectType, since, newLibraryVersion, options = {}) {
	var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
	var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
	
	// Get versions of all objects updated remotely since the current local library version
	Zotero.debug(`Checking for updated ${options.top ? 'top-level ' : ''}`
		+ `${objectTypePlural} in ${this.library.name}`);
	var queryParams = {};
	if (since) {
		queryParams.since = since;
	}
	if (options.top) {
		queryParams.top = true;
	}
	var results = await this.apiClient.getVersions(
		this.library.libraryType,
		this.libraryTypeID,
		objectType,
		queryParams
	);
	
	Zotero.debug("VERSIONS:");
	Zotero.debug(JSON.stringify(results));
	
	// If something else modified the remote library while we were getting updates,
	// wait for increasing amounts of time before trying again, and then start from
	// the beginning
	if (newLibraryVersion != results.libraryVersion) {
		return this.DOWNLOAD_RESULT_RESTART;
	}
	
	
	var numObjects = Object.keys(results.versions).length;
	if (numObjects) {
		Zotero.debug(numObjects + " " + (numObjects == 1 ? objectType : objectTypePlural)
			+ " modified since last check");
	}
	else {
		Zotero.debug("No " + objectTypePlural + " modified remotely since last check");
	}
	
	// Get objects that should be retried based on the current time, unless it's top-level items mode.
	// (We don't know if the queued items are top-level or not, so we do them with child items.)
	let queuedKeys = [];
	if (objectType != 'item' || !options.top) {
		if (this.firstInSession || !this.background) {
			queuedKeys = await Zotero.Sync.Data.Local.getObjectsFromSyncQueue(
				objectType, this.libraryID
			);
		}
		else {
			queuedKeys = await Zotero.Sync.Data.Local.getObjectsToTryFromSyncQueue(
				objectType, this.libraryID
			);
		}
		// Don't include items that just failed in the top-level run
		if (this.failedItems.length) {
			queuedKeys = Zotero.Utilities.arrayDiff(queuedKeys, this.failedItems);
		}
		if (queuedKeys.length) {
			Zotero.debug(`Refetching ${queuedKeys.length} queued `
				+ (queuedKeys.length == 1 ? objectType : objectTypePlural))
		}
	}
	
	if (!numObjects && !queuedKeys.length) {
		return false;
	}
	
	let keys = [];
	let versions = await objectsClass.getObjectVersions(
		this.libraryID, Object.keys(results.versions)
	);
	let upToDate = [];
	for (let key in results.versions) {
		// Skip objects that are already up-to-date. Generally all returned objects should have
		// newer version numbers, but there are some situations, such as full syncs or
		// interrupted syncs, where we may get versions for objects that are already up-to-date
		// locally.
		if (versions[key] == results.versions[key]) {
			upToDate.push(key);
			continue;
		}
		keys.push(key);
	}
	if (upToDate.length) {
		Zotero.debug(`Skipping up-to-date ${objectTypePlural} in library ${this.libraryID}: `
			+ upToDate.sort().join(", "));
	}
	
	// In child-items mode, remove top-level items that just failed
	if (objectType == 'item' && !options.top && this.failedItems.length) {
		keys = Zotero.Utilities.arrayDiff(keys, this.failedItems);
	}
	
	keys.push(...queuedKeys);
	keys = Zotero.Utilities.arrayUnique(keys);
	
	if (!keys.length) {
		Zotero.debug(`No ${objectTypePlural} to download`);
		return this.DOWNLOAD_RESULT_CONTINUE;
	}
	
	return this._downloadObjects(objectType, keys);
};


/**
 * Download data for specified objects from the API and run processing on them, and show the conflict
 * resolution window if necessary
 *
 * @return {Promise<Integer>} - A download result code (this.DOWNLOAD_RESULT_*)
 */
Zotero.Sync.Data.Engine.prototype._downloadObjects = async function (objectType, keys) {
	var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
	
	var remainingKeys = [...keys];
	var lastLength = keys.length;
	var objectData = {};
	keys.forEach(key => objectData[key] = null);
	
	while (true) {
		this._statusCheck();
		
		// Get data we've downloaded in a previous loop but failed to process
		var results = [];
		let keysToDownload = [];
		var keysToReprocess = [];
		for (let key in objectData) {
			if (objectData[key] === null) {
				keysToDownload.push(key);
			}
			else {
				keysToReprocess.push(key);
				
			}
		}
		if (keysToReprocess) {
			results.push({
				keys: keysToReprocess,
				json: keysToReprocess.map(key => objectData[key])
			});
		}
		// Add promises for batches of downloaded data for remaining keys
		results.push(...this.apiClient.downloadObjects(
			this.library.libraryType,
			this.libraryTypeID,
			objectType,
			keysToDownload
		));
		
		// TODO: localize
		this.setStatus(
			"Downloading "
			+ (keysToDownload.length == 1
				? "1 " + objectType
				: Zotero.Utilities.numberFormat(remainingKeys.length, 0) + " " + objectTypePlural)
			+ " in " + this.library.name
		);
		
		var missingKeys = [];
		var conflicts = [];
		var restored = [];
		var num = 0;
		
		// Process batches of object data as they're available, one at a time
		await Array.fromAsync(results, async function ({ keys: batchKeys, json, error }) {
			this._statusCheck();
			
			Zotero.debug(`Processing batch of downloaded ${objectTypePlural} in ${this.library.name}`);
			
			if (error) {
				this.failed = error;
				return;
			}
			
			// Save downloaded JSON for later attempts
			var seenKeys = new Set();
			json.forEach(obj => {
				objectData[obj.key] = obj;
				seenKeys.add(obj.key);
			});
			// If a key we requested wasn't returned, it's missing remotely
			for (let key of batchKeys) {
				if (!seenKeys.has(key)) {
					missingKeys.push(key);
					objectData[key] = false;
				}
			}
			
			// Process objects
			let results = await Zotero.Sync.Data.Local.processObjectsFromJSON(
				objectType,
				this.libraryID,
				json,
				this._getOptions({
					onObjectProcessed: () => {
						num++;
						// Check for stop every 5 items
						if (num % 5 == 0) {
							this._statusCheck();
						}
					},
					// Increase the notifier batch size as we go, so that new items start coming in
					// one by one but then switch to larger chunks
					getNotifierBatchSize: () => {
						var size;
						if (num < 10) {
							size = 1;
						}
						else if (num < 50) {
							size = 5;
						}
						else if (num < 150) {
							size = 25;
						}
						else {
							size = 50;
						}
						return Math.min(size, json.length);
					}
				})
			);
			
			num += results.length;
			let processedKeys = [];
			let conflictResults = [];
			results.forEach(x => {
				// If data was processed, remove JSON
				if (x.processed) {
					delete objectData[x.key];
					
					// We'll need to add items back to restored collections
					if (x.restored) {
						restored.push(x.key);
					}
				}
				// If object shouldn't be retried, mark as processed
				if (x.processed || !x.retry) {
					processedKeys.push(x.key);
				}
				if (x.conflict) {
					conflictResults.push(x);
				}
			});
			remainingKeys = Zotero.Utilities.arrayDiff(remainingKeys, processedKeys);
			conflicts.push(...conflictResults);
		}.bind(this));
		
		// If any locally deleted collections were restored, either add them back to the collection
		// (if the items still exist) or remove them from the delete log and add them to the sync queue
		if (restored.length && objectType == 'collection') {
			await this._restoreRestoredCollectionItems(restored);
		}
		
		this._statusCheck();
		
		// If all requests were successful, such that we had a chance to see all keys, remove keys we
		// didn't see from the sync queue so they don't keep being retried forever
		if (missingKeys.length) {
			Zotero.debug(`Removing ${missingKeys.length} missing `
				+ Zotero.Utilities.pluralize(missingKeys.length, [objectType, objectTypePlural])
				+ " from sync queue");
			await Zotero.Sync.Data.Local.removeObjectsFromSyncQueue(objectType, this.libraryID, missingKeys);
			remainingKeys = Zotero.Utilities.arrayDiff(remainingKeys, missingKeys);
		}
		
		if (!remainingKeys.length || remainingKeys.length == lastLength) {
			// Add failed objects to sync queue
			//
			// Object failed if it's still present in the object data (since successfully processed
			// objects are removed) and it's not false (meaning it was missing from the request,
			// which isn't a failure)
			let failedKeys = keys.filter(key => objectData[key] !== undefined && objectData[key] !== false);
			if (failedKeys.length) {
				Zotero.debug(`Queueing ${failedKeys.length} failed `
					+ Zotero.Utilities.pluralize(failedKeys.length, [objectType, objectTypePlural])
					+ " for later", 2);
				await Zotero.Sync.Data.Local.addObjectsToSyncQueue(
					objectType, this.libraryID, failedKeys
				);
				
				// Note failed item keys so child items step (if this isn't it) can skip them
				if (objectType == 'item') {
					this.failedItems = failedKeys;
				}
			}
			else {
				Zotero.debug(`All ${objectTypePlural} for ${this.library.name} saved to database`);
				
				if (objectType == 'item') {
					this.failedItems = [];
				}
			}
			break;
		}
		
		lastLength = remainingKeys.length;
		
		Zotero.debug(`Retrying ${remainingKeys.length} remaining `
			+ Zotero.Utilities.pluralize(remainingKeys, [objectType, objectTypePlural]));
	}
	
	// Show conflict resolution window
	if (conflicts.length) {
		this._statusCheck();
		
		let results = await Zotero.Sync.Data.Local.processConflicts(
			objectType, this.libraryID, conflicts, this._getOptions()
		);
		let keys = results.filter(x => x.processed).map(x => x.key);
		// If all keys are unprocessed and didn't fail from an error, conflict resolution was cancelled
		if (results.every(x => !x.processed && !x.error)) {
			throw new Zotero.Sync.UserCancelledException();
		}
		await Zotero.Sync.Data.Local.removeObjectsFromSyncQueue(objectType, this.libraryID, keys);
	}
	
	return this.DOWNLOAD_RESULT_CONTINUE;
};


/**
 * If a collection is deleted locally but modified remotely between syncs, the local collection is
 * restored, but collection membership is a property of items, the local items that were previously
 * in that collection won't be any longer (or they might have been deleted along with the collection),
 * so we have to get the current collection items from the API and either add them back
 * (if they exist) or clear them from the delete log and mark them for download.
 *
 * Remote items in the trash aren't currently restored and will be removed from the collection when the
 * local collection-item removal syncs up.
 */
Zotero.Sync.Data.Engine.prototype._restoreRestoredCollectionItems = async function (collectionKeys) {
	for (let collectionKey of collectionKeys) {
		let { keys: itemKeys } = await this.apiClient.getKeys(
			this.library.libraryType,
			this.libraryTypeID,
			{
				target: `collections/${collectionKey}/items/top`,
				format: 'keys'
			}
		);
		
		if (itemKeys.length) {
			let collection = Zotero.Collections.getByLibraryAndKey(this.libraryID, collectionKey);
			let addToCollection = [];
			let addToQueue = [];
			for (let itemKey of itemKeys) {
				let o = Zotero.Items.getByLibraryAndKey(this.libraryID, itemKey);
				if (o) {
					addToCollection.push(o.id);
					// Remove item from trash if it's there, since it's not in the trash remotely.
					// (This would happen if items were moved to the trash along with the collection
					// deletion.)
					if (o.deleted) {
						o.deleted = false
						await o.saveTx();
					}
				}
				else {
					addToQueue.push(itemKey);
				}
			}
			if (addToCollection.length) {
				Zotero.debug(`Restoring ${addToCollection.length} `
					+ `${Zotero.Utilities.pluralize(addToCollection.length, ['item', 'items'])} `
					+ `to restored collection ${collection.libraryKey}`);
				await Zotero.DB.executeTransaction(async function () {
					await collection.addItems(addToCollection);
				}.bind(this));
			}
			if (addToQueue.length) {
				Zotero.debug(`Restoring ${addToQueue.length} deleted `
					+ `${Zotero.Utilities.pluralize(addToQueue.length, ['item', 'items'])} `
					+ `in restored collection ${collection.libraryKey}`);
				await Zotero.Sync.Data.Local.removeObjectsFromDeleteLog(
					'item', this.libraryID, addToQueue
				);
				await Zotero.Sync.Data.Local.addObjectsToSyncQueue(
					'item', this.libraryID, addToQueue
				);
			}
		}
	}
};



/**
 * Get deleted objects from the API and process them
 *
 * @param {Integer} since - Last-known library version; get changes sinces this version
 * @param {Integer} [newLibraryVersion] - Newest library version seen in this sync process; if newer
 *     version is seen, restart the sync
 * @return {Object} - Object with 'result' (DOWNLOAD_RESULT_*) and 'libraryVersion'
 */
Zotero.Sync.Data.Engine.prototype._downloadDeletions = async function (since, newLibraryVersion) {
	const batchSize = 50;
	
	let results = await this.apiClient.getDeleted(
		this.library.libraryType,
		this.libraryTypeID,
		since
	);
	if (newLibraryVersion && newLibraryVersion != results.libraryVersion) {
		return {
			result: this.DOWNLOAD_RESULT_RESTART,
			libraryVersion: results.libraryVersion
		};
	}
	
	var numObjects = Object.keys(results.deleted).reduce((n, k) => n + results.deleted[k].length, 0);
	if (!numObjects) {
		Zotero.debug("No objects deleted remotely since last check");
		return {
			result: this.DOWNLOAD_RESULT_CONTINUE,
			libraryVersion: results.libraryVersion
		};
	}
	
	Zotero.debug(numObjects + " objects deleted remotely since last check");
	
	// Process deletions
	for (let objectTypePlural in results.deleted) {
		let objectType = Zotero.DataObjectUtilities.getObjectTypeSingular(objectTypePlural);
		let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
		let toDelete = [];
		let conflicts = [];
		for (let key of results.deleted[objectTypePlural]) {
			// TODO: Remove from request?
			if (objectType == 'tag') {
				continue;
			}
			
			if (objectType == 'setting') {
				let meta = Zotero.SyncedSettings.getMetadata(this.libraryID, key);
				if (!meta) {
					continue;
				}
				if (meta.synced) {
					await Zotero.SyncedSettings.clear(this.libraryID, key, {
						skipDeleteLog: true
					});
				}
				
				// Ignore setting if changed locally
				continue;
			}
			
			let obj = objectsClass.getByLibraryAndKey(this.libraryID, key);
			if (!obj) {
				continue;
			}
			if (obj.synced) {
				toDelete.push(obj);
			}
			// Conflict resolution
			else if (objectType == 'item') {
				// If item is already in trash locally, just delete it
				if (obj.deleted) {
					Zotero.debug("Local item is in trash -- applying remote deletion");
					await obj.eraseTx({
						skipDeleteLog: true
					});
					continue;
				}
				conflicts.push({
					libraryID: this.libraryID,
					left: obj.toJSON(),
					right: {
						deleted: true
					}
				});
			}
		}
		
		if (conflicts.length) {
			this._statusCheck();
			
			// Sort conflicts by Date Modified
			conflicts.sort(function (a, b) {
				var d1 = a.left.dateModified;
				var d2 = b.left.dateModified;
				if (d1 > d2) {
					return 1
				}
				if (d1 < d2) {
					return -1;
				}
				return 0;
			});
			var mergeData = Zotero.Sync.Data.Local.showConflictResolutionWindow(conflicts);
			if (!mergeData) {
				Zotero.debug("Cancelling sync");
				throw new Zotero.Sync.UserCancelledException();
			}
			await Zotero.Utilities.Internal.forEachChunkAsync(
				mergeData,
				batchSize,
				function (chunk) {
					return Zotero.DB.executeTransaction(async function () {
						for (let json of chunk) {
							let data = json.data;
							if (!data.deleted) continue;
							let obj = objectsClass.getByLibraryAndKey(this.libraryID, data.key);
							if (!obj) {
								Zotero.logError("Remotely deleted " + objectType
									+ " didn't exist after conflict resolution");
								continue;
							}
							await obj.erase({
								skipEditCheck: true
							});
						}
					}.bind(this));
				}.bind(this)
			);
		}
		
		if (toDelete.length) {
			await Zotero.Utilities.Internal.forEachChunkAsync(
				toDelete,
				batchSize,
				function (chunk) {
					return Zotero.DB.executeTransaction(async function () {
						for (let obj of chunk) {
							await obj.erase({
								skipEditCheck: true,
								skipDeleteLog: true
							});
						}
					});
				}
			);
		}
	}
	
	return {
		result: this.DOWNLOAD_RESULT_CONTINUE,
		libraryVersion: results.libraryVersion
	};
};


/**
 * If something else modified the remote library while we were getting updates, wait for increasing
 * amounts of time before trying again, and then start from the beginning
 */
Zotero.Sync.Data.Engine.prototype._onLibraryVersionChange = async function (mode) {
	Zotero.logError("Library version changed since last download -- restarting sync");
	
	if (!this.downloadDelayGenerator) {
		this.downloadDelayGenerator = Zotero.Utilities.Internal.delayGenerator(
			Zotero.Sync.Data.conflictDelayIntervals, 60 * 60 * 1000
		);
	}
	
	let keepGoing = await this.downloadDelayGenerator.next().value;
	if (!keepGoing) {
		throw new Error("Could not update " + this.library.name + " -- library in use");
	}
};


/**
 * Get unsynced objects, build upload JSON, and start API requests
 *
 * @throws {Zotero.HTTP.UnexpectedStatusException}
 * @return {Promise<Integer>} - An upload result code (this.UPLOAD_RESULT_*)
 */
Zotero.Sync.Data.Engine.prototype._startUpload = async function () {
	var libraryVersion = this.library.libraryVersion;
	
	var settingsUploaded = false;
	var uploadNeeded = false;
	var objectIDs = {};
	var objectDeletions = {};
	
	// Upload synced settings
	try {
		let settings = await Zotero.SyncedSettings.getUnsynced(this.libraryID);
		if (Object.keys(settings).length) {
			libraryVersion = await this._uploadSettings(settings, libraryVersion);
			settingsUploaded = true;
		}
		else {
			Zotero.debug("No settings to upload in " + this.library.name);
		}
	}
	catch (e) {
		return this._handleUploadError(e);
	}
	
	// Get unsynced local objects for each object type
	for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) {
		this._statusCheck();
		
		let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
		let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
		
		// New/modified objects
		let ids = await Zotero.Sync.Data.Local.getUnsynced(objectType, this.libraryID);
		
		// Skip objects in sync queue, because they might have unresolved conflicts.
		// The queue only has keys, so we have to convert to keys and back.
		let unsyncedKeys = ids.map(id => objectsClass.getLibraryAndKeyFromID(id).key);
		let origUnsynced = unsyncedKeys; // TEMP
		let queueKeys = await Zotero.Sync.Data.Local.getObjectsFromSyncQueue(objectType, this.libraryID);
		let newUnsyncedKeys = Zotero.Utilities.arrayDiff(unsyncedKeys, queueKeys);
		if (newUnsyncedKeys.length < unsyncedKeys.length) {
			Zotero.debug(`Skipping ${unsyncedKeys.length - newUnsyncedKeys.length} key(s) in sync queue`);
			Zotero.debug(Zotero.Utilities.arrayDiff(unsyncedKeys, newUnsyncedKeys));
		}
		unsyncedKeys = newUnsyncedKeys;
		
		// TEMP
		//ids = unsyncedKeys.map(key => objectsClass.getIDFromLibraryAndKey(this.libraryID, key));
		let missing = [];
		ids = unsyncedKeys.map(key => {
			let id = objectsClass.getIDFromLibraryAndKey(this.libraryID, key)
			if (!id) {
				Zotero.debug("Missing id for key " + key);
				missing.push(key);
			}
			return id;
		});
		if (missing.length) {
			Zotero.debug("Missing " + objectTypePlural + ":");
			Zotero.debug(ids.filter(id => !id));
			Zotero.debug(ids.filter(id => !objectsClass.getLibraryAndKeyFromID(id)));
			Zotero.debug(missing);
		}
		
		if (ids.length) {
			Zotero.debug(ids.length + " "
				+ (ids.length == 1 ? objectType : objectTypePlural)
				+ " to upload in library " + this.libraryID);
			objectIDs[objectType] = ids;
		}
		else {
			Zotero.debug("No " + objectTypePlural + " to upload in " + this.library.name);
		}
		
		// Deleted objects
		let keys = await Zotero.Sync.Data.Local.getDeleted(objectType, this.libraryID);
		if (keys.length) {
			Zotero.debug(`${keys.length} ${objectType} deletion`
				+ (keys.length == 1 ? '' : 's')
				+ ` to upload in ${this.library.name}`);
			objectDeletions[objectType] = keys;
		}
		else {
			Zotero.debug(`No ${objectType} deletions to upload in ${this.library.name}`);
		}
		
		if (ids.length || keys.length) {
			uploadNeeded = true;
		}
	}
	
	if (!uploadNeeded) {
		return settingsUploaded ? this.UPLOAD_RESULT_SUCCESS : this.UPLOAD_RESULT_NOTHING_TO_UPLOAD;
	}
	
	try {
		Zotero.debug(JSON.stringify(objectIDs));
		for (let objectType in objectIDs) {
			this._statusCheck();
			
			libraryVersion = await this._uploadObjects(
				objectType, objectIDs[objectType], libraryVersion
			);
		}
		
		Zotero.debug(JSON.stringify(objectDeletions));
		for (let objectType in objectDeletions) {
			this._statusCheck();
			
			libraryVersion = await this._uploadDeletions(
				objectType, objectDeletions[objectType], libraryVersion
			);
		}
	}
	catch (e) {
		return this._handleUploadError(e);
	}
	
	return this.UPLOAD_RESULT_SUCCESS;
};


Zotero.Sync.Data.Engine.prototype._uploadSettings = async function (settings, libraryVersion) {
	const SETTINGS_BATCH_SIZE = 250;
	await Zotero.Utilities.Internal.forEachChunkAsync(
		Object.keys(settings),
		SETTINGS_BATCH_SIZE,
		async function (keys) {
			let json = {};
			for (let key of keys) {
				json[key] = {
					value: settings[key]
				};
			}
			libraryVersion = await this.apiClient.uploadSettings(
				this.library.libraryType,
				this.libraryTypeID,
				libraryVersion,
				json
			);
			await Zotero.SyncedSettings.markAsSynced(
				this.libraryID,
				keys,
				libraryVersion
			);
			
			if (this.library.libraryVersion == this.library.storageVersion) {
				this.library.storageVersion = libraryVersion;
			}
			this.library.libraryVersion = libraryVersion;
			await this.library.saveTx({
				skipNotifier: true
			});
		}.bind(this)
	);
	Zotero.debug("Done uploading settings in " + this.library.name);
	return libraryVersion;
};


Zotero.Sync.Data.Engine.prototype._uploadObjects = async function (objectType, ids, libraryVersion) {
	let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
	let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
	
	let queue = [];
	for (let id of ids) {
		queue.push({
			id: id,
			json: null,
			tries: 0,
			failed: false
		});
	}
	
	// Watch for objects that change locally during the sync, so that we don't overwrite them with the
	// older saved server version after uploading
	var changedObjects = new Set();
	var observerID = Zotero.Notifier.registerObserver(
		{
			notify: function (event, type, ids, extraData) {
				let keys = [];
				if (event == 'modify') {
					keys = ids.map(id => {
						var { libraryID, key } = objectsClass.getLibraryAndKeyFromID(id);
						return (libraryID == this.libraryID) ? key : false;
					});
				}
				else if (event == 'delete') {
					keys = ids.map(id => {
						if (!extraData[id]) return false;
						var { libraryID, key } = extraData[id];
						return (libraryID == this.libraryID) ? key : false;
					});
				}
				keys.filter(key => key).forEach(key => changedObjects.add(key));
			}.bind(this)
		},
		[objectType],
		objectTypePlural + "Upload"
	);
	
	try {
		while (queue.length) {
			this._statusCheck();
			
			// Get a slice of the queue and generate JSON for objects if necessary
			let batch = [];
			let numSkipped = 0;
			for (let i = 0; i < queue.length && i < this.uploadBatchSize; i++) {
				let o = queue[i];
				// Skip requests that failed with 4xx or that have been retried too many times
				if (o.failed || o.tries >= this.maxUploadTries) {
					numSkipped++;
					continue;
				}
				if (!o.json) {
					o.json = await this._getJSONForObject(
						objectType,
						o.id,
						{
							restoreToServer: this._restoringToServer,
							// Only include storage properties ('mtime', 'md5') when restoring to
							// server and for WebDAV files
							skipStorageProperties:
								objectType == 'item'
									? !this._restoringToServer
										&& Zotero.Sync.Storage.Local.getModeForLibrary(this.library.libraryID) != 'webdav'
									: undefined
						}
					);
				}
				batch.push(o);
			}
			
			// No more non-failed requests
			if (!batch.length) {
				Zotero.debug(`No more ${objectTypePlural} to upload`);
				break;
			}
			
			// Remove selected and skipped objects from queue
			queue.splice(0, batch.length + numSkipped);
			
			let jsonBatch = batch.map(o => o.json);
			
			Zotero.debug("UPLOAD BATCH:");
			Zotero.debug(jsonBatch);
			
			let results;
			let numSuccessful = 0;
			({ libraryVersion, results } = await this.apiClient.uploadObjects(
				this.library.libraryType,
				this.libraryTypeID,
				"POST",
				libraryVersion,
				objectType,
				jsonBatch
			));
			
			// Mark successful and unchanged objects as synced with new version,
			// and save uploaded JSON to cache
			let updateVersionIDs = [];
			let updateSyncedIDs = [];
			let toSave = [];
			let toCache = [];
			for (let state of ['successful', 'unchanged']) {
				for (let index in results[state]) {
					let current = results[state][index];
					// 'successful' includes objects, not keys
					let key = state == 'successful' ? current.key : current;
					let changed = changedObjects.has(key);
					
					if (key != jsonBatch[index].key) {
						throw new Error("Key mismatch (" + key + " != " + jsonBatch[index].key + ")");
					}
					
					let obj = objectsClass.getByLibraryAndKey(this.libraryID, key);
					// This might not exist if the object was deleted during the upload
					if (obj) {
						updateVersionIDs.push(obj.id);
						if (!changed) {
							updateSyncedIDs.push(obj.id);
						}
					}
					
					// Successful
					if (state == 'successful') {
						// Update local object with saved data if necessary, as long as it hasn't
						// changed locally since the upload
						if (!changed) {
							obj.fromJSON(current.data, { strict: true });
							toSave.push(obj);
						}
						else {
							Zotero.debug("Local version changed during upload "
								+ "-- not updating from remotely saved version");
						}
						toCache.push(current);
					}
					// Unchanged
					else {
						// If a cache object exists, there shouldn't be any need to update it, since
						// nothing was changed on the server, including the version.
						//
						// If there's no cache object, we can save the uploaded object, which we know
						// matches what's on the server. The version number won't necessarily reflect
						// the actual version of the object on the server, since objects are uploaded
						// in batches and we only get the final version, but it will guarantee that
						// the object won't be redownloaded unnecessarily in the case of a full sync,
						// because the version will be higher than whatever version is on the server.
						let hasCacheObject = !!Zotero.Sync.Data.Local.getCacheObject(
							objectType, obj.libraryID, obj.key, obj.version
						);
						if (!hasCacheObject) {
							jsonBatch[index].version = libraryVersion;
							toCache.push(jsonBatch[index]);
						}
					}
					
					numSuccessful++;
					// Remove from batch to mark as successful
					delete batch[index];
					delete jsonBatch[index];
				}
			}
			await Zotero.Sync.Data.Local.saveCacheObjects(
				objectType, this.libraryID, toCache
			);
			await Zotero.DB.executeTransaction(async function () {
				for (let i = 0; i < toSave.length; i++) {
					await toSave[i].save({
						skipSelect: true,
						skipSyncedUpdate: true,
						// We want to minimize the times when server writes actually result in local
						// updates, but when they do, don't update the user-visible timestamp
						skipDateModifiedUpdate: true,
						// Replace Online Library can reupload annotations created by others in a
						// group library, and the server could theoretically override a value, so
						// allow updating local annotations even if they were created by someone
						// else. There might be other cases where this is necessary, but let's start
						// with this.
						skipEditCheck: objectType == 'item'
							&& this.library.isGroup
							&& toSave[i].isAnnotation()
					});
				}
				if (this.library.libraryVersion == this.library.storageVersion) {
					this.library.storageVersion = libraryVersion;
				}
				this.library.libraryVersion = libraryVersion;
				await this.library.save();
				objectsClass.updateVersion(updateVersionIDs, libraryVersion);
				objectsClass.updateSynced(updateSyncedIDs, true);
			}.bind(this));
			
			// Purge older objects in sync cache
			if (toSave.length) {
				await Zotero.Sync.Data.Local.purgeCache(objectType, this.libraryID);
			}
			
			// Handle failed objects
			for (let index in results.failed) {
				let { code, message, data } = results.failed[index];
				let key = jsonBatch[index].key;
				// API errors are HTML
				message = Zotero.Utilities.unescapeHTML(message);
				let e = new Error(message);
				e.name = "ZoteroObjectUploadError";
				e.code = code;
				if (data) {
					e.data = data;
				}
				e.objectType = objectType;
				e.object = objectsClass.getByLibraryAndKey(this.libraryID, key);
				
				Zotero.logError(`Error ${code} for ${objectType} ${key} in `
					+ this.library.name + ":\n\n" + e);
				
				let keepGoing = await this._checkObjectUploadError(objectType, key, e, queue, batch);
				if (keepGoing) {
					numSuccessful++;
					continue;
				}
				
				if (this.onError) {
					this.onError(e);
				}
				if (this.stopOnError) {
					throw e;
				}
				batch[index].tries++;
				// Mark 400 errors as permanently failed
				if (e.code < 500) {
					batch[index].failed = true;
				}
				// 500 errors should stay in queue and be retried, unless a dependency also failed
				else if (objectType == 'item') {
					// Check parent item
					let parentItem = batch[index].json.parentItem;
					if (parentItem) {
						for (let i in batch) {
							if (i == index) break;
							let o = batch[i];
							if (o.failed && o.json.key) {
								Zotero.debug(`Not retrying child of failed parent ${parentItem}`);
								batch[index].failed = true;
							}
						}
					}
				}
			}
			
			// Add failed objects back to end of queue
			var numFailed = 0;
			for (let o of batch) {
				if (o !== undefined) {
					queue.push(o);
					// TODO: Clear JSON?
					numFailed++;
				}
			}
			Zotero.debug("Failed: " + numFailed, 2);
			
			// If we didn't make any progress, bail
			if (!numSuccessful) {
				throw new Error("Made no progress during upload -- stopping");
			}
		}
	}
	finally {
		Zotero.Notifier.unregisterObserver(observerID);
	}
	Zotero.debug("Done uploading " + objectTypePlural + " in library " + this.libraryID);
	
	return libraryVersion;
}


Zotero.Sync.Data.Engine.prototype._uploadDeletions = async function (objectType, keys, libraryVersion) {
	let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
	let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
	
	while (keys.length) {
		let batch = keys.slice(0, this.uploadDeletionBatchSize);
		libraryVersion = await this.apiClient.uploadDeletions(
			this.library.libraryType,
			this.libraryTypeID,
			libraryVersion,
			objectType,
			batch
		);
		keys.splice(0, batch.length);
		
		// Update library version
		if (this.library.libraryVersion == this.library.storageVersion) {
			this.library.storageVersion = libraryVersion;
		}
		this.library.libraryVersion = libraryVersion;
		await this.library.saveTx({
			skipNotifier: true
		});
		
		// Remove successful deletions from delete log
		await Zotero.Sync.Data.Local.removeObjectsFromDeleteLog(
			objectType, this.libraryID, batch
		);
	}
	Zotero.debug(`Done uploading ${objectType} deletions in ${this.library.name}`);
	
	return libraryVersion;
};


/**
 * Update createdByUserID/lastModifiedByUserID for previously downloaded group items
 *
 * TEMP: Currently only processes one batch of items, but before we start displaying the names,
 * we'll need to update it to fetch all
 */
Zotero.Sync.Data.Engine.prototype._updateGroupItemUsers = async function () {
	// TODO: Do more at once when we actually start showing these names
	var max = this.apiClient.MAX_OBJECTS_PER_REQUEST;
	
	var sql = "SELECT key FROM items LEFT JOIN groupItems GI USING (itemID) "
		+ `WHERE libraryID=? AND GI.itemID IS NULL ORDER BY itemID LIMIT ${max}`;
	var keys = await Zotero.DB.columnQueryAsync(sql, this.libraryID);
	if (!keys.length) {
		return;
	}
	
	Zotero.debug(`Updating item users in ${this.library.name}`);
	
	var { json: jsonItems, error } = await this.apiClient.downloadObjects(
		this.library.libraryType, this.libraryTypeID, 'item', keys
	)[0];
	
	if (error) {
		Zotero.logError(error);
		return;
	}
	
	for (let jsonItem of jsonItems) {
		let item = Zotero.Items.getByLibraryAndKey(this.libraryID, jsonItem.key);
		let params = [null, null];
		
		// This should almost always exist, but maybe doesn't for some old items?
		if (jsonItem.meta.createdByUser) {
			let { id: userID, username, name } = jsonItem.meta.createdByUser;
			await Zotero.Users.setName(userID, name !== '' ? name : username);
			params[0] = userID;
		}
		
		if (jsonItem.meta.lastModifiedByUser) {
			let { id: userID, username, name } = jsonItem.meta.lastModifiedByUser;
			await Zotero.Users.setName(userID, name !== '' ? name : username);
			params[1] = userID;
		}
		
		await item.updateCreatedByUser.apply(item, params);
	}
	
	return;
};


Zotero.Sync.Data.Engine.prototype._getJSONForObject = function (objectType, id, options = {}) {
	return Zotero.DB.executeTransaction(async function () {
		var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
		var obj = objectsClass.get(id);
		var cacheObj = false;
		// If the object has been synced before, get the pristine version from the cache so we can
		// use PATCH mode and include only fields that have changed
		if (obj.version) {
			cacheObj = await Zotero.Sync.Data.Local.getCacheObject(
				objectType, obj.libraryID, obj.key, obj.version
			);
		}
		var patchBase = false;
		// If restoring to server, use full mode. (The version and cache are cleared, so we would
		// use "new" otherwise, which might be slightly different.)
		if (options.restoreToServer) {
			var mode = 'full';
		}
		// If copy of object in cache, use patch mode with cache data as the base
		else if (cacheObj) {
			var mode = 'patch';
			patchBase = cacheObj.data;
		}
		// Otherwise use full mode if there's a version
		else {
			var mode = obj.version ? "full" : "new";
		}
		return obj.toJSON({
			mode,
			includeKey: true,
			includeVersion: !options.restoreToServer,
			includeDate: true,
			// Whether to skip 'mtime' and 'md5'
			skipStorageProperties: options.skipStorageProperties,
			// Use last-synced mtime/md5 instead of current values from the file itself
			syncedStorageProperties: true,
			patchBase
		});
	});
}


/**
 * Upgrade library to current sync architecture
 *
 * This sets the 'synced' and 'version' properties based on classic last-sync times and object
 * modification times. Objects are marked as:
 *
 * - synced=1 if modified locally before the last classic sync time
 * - synced=0 (unchanged) if modified locally since the last classic sync time
 * - version=<remote version> if modified remotely before the last classic sync time
 * - version=0 if modified remotely since the last classic sync time
 *
 * If both are 0, that's a conflict.
 *
 * @return {Object[]} - Objects returned from getVersions(), keyed by objectType, for use
 *                      by _fullSync()
 */
Zotero.Sync.Data.Engine.prototype._upgradeCheck = async function () {
	var libraryVersion = this.library.libraryVersion;
	if (libraryVersion) return;
	
	var lastLocalSyncTime = await Zotero.DB.valueQueryAsync(
		"SELECT version FROM version WHERE schema='lastlocalsync'"
	);
	// Never synced with classic architecture, or already upgraded and full sync (which updates
	// library version) didn't finish
	if (!lastLocalSyncTime) return;
	
	Zotero.debug("Upgrading library to current sync architecture");
	
	var lastRemoteSyncTime = await Zotero.DB.valueQueryAsync(
		"SELECT version FROM version WHERE schema='lastremotesync'"
	);
	// Shouldn't happen
	if (!lastRemoteSyncTime) lastRemoteSyncTime = lastLocalSyncTime;
	
	var objectTypes = Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID);
	
	// Mark all items modified locally before the last classic sync time as synced
	if (lastLocalSyncTime) {
		lastLocalSyncTime = new Date(lastLocalSyncTime * 1000);
		for (let objectType of objectTypes) {
			let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
			let ids = await objectsClass.getOlder(this.libraryID, lastLocalSyncTime);
			await objectsClass.updateSynced(ids, true);
		}
	}
	
	var versionResults = {};
	var currentVersions = {};
	var gen;
	loop:
	while (true) {
		let lastLibraryVersion = 0;
		for (let objectType of objectTypes) {
			currentVersions[objectType] = {};
			
			let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
			 // TODO: localize
			this.setStatus("Updating " + objectTypePlural + " in " + this.library.name);
			
			// Get versions from API for all objects
			let allResults = await this.apiClient.getVersions(
				this.library.libraryType,
				this.libraryTypeID,
				objectType
			);
			
			// Get versions from API for objects modified remotely since the last classic sync time
			let sinceResults = await this.apiClient.getVersions(
				this.library.libraryType,
				this.libraryTypeID,
				objectType,
				{
					sincetime: lastRemoteSyncTime
				}
			);
			
			// If something else modified the remote library while we were getting updates,
			// wait for increasing amounts of time before trying again, and then start from
			// the first object type
			if (allResults.libraryVersion != sinceResults.libraryVersion
					|| (lastLibraryVersion && allResults.libraryVersion != lastLibraryVersion)) {
				if (!gen) {
					gen = Zotero.Utilities.Internal.delayGenerator(
						Zotero.Sync.Data.conflictDelayIntervals, 60 * 60 * 1000
					);
				}
				Zotero.debug("Library version changed since last check ("
					+ allResults.libraryVersion + " != "
					+ sinceResults.libraryVersion + " != "
					+ lastLibraryVersion + ") -- waiting");
				let keepGoing = await gen.next().value;
				if (!keepGoing) {
					throw new Error("Could not update " + this.library.name + " -- library in use");
				}
				continue loop;
			}
			else {
				lastLibraryVersion = allResults.libraryVersion;
			}
			
			versionResults[objectType] = allResults;
			
			// Get versions for remote objects modified remotely before the last classic sync time,
			// which is all the objects not modified since that time
			for (let key in allResults.versions) {
				if (!sinceResults.versions[key]) {
					currentVersions[objectType][key] = allResults.versions[key];
				}
			}
		}
		break;
	}
	
	// Update versions on local objects modified remotely before last classic sync time,
	// to indicate that they don't need to receive remote updates
	await Zotero.DB.executeTransaction(async function () {
		for (let objectType in currentVersions) {
			let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
			let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
			
			// TODO: localize
			this.setStatus("Updating " + objectTypePlural + " in " + this.library.name);
			
			// Group objects with the same version together and update in batches
			let versionObjects = {};
			for (let key in currentVersions[objectType]) {
				let id = objectsClass.getIDFromLibraryAndKey(this.libraryID, key);
				// If local object doesn't exist, skip
				if (!id) continue;
				let version = currentVersions[objectType][key];
				if (!versionObjects[version]) {
					versionObjects[version] = [];
				}
				versionObjects[version].push(id);
			}
			for (let version in versionObjects) {
				await objectsClass.updateVersion(versionObjects[version], version);
			}
		}
		
		// Mark library as requiring full sync
		this.library.libraryVersion = -1;
		await this.library.save();
		
		// If this is the last classic sync library, delete old timestamps
		if (!(await Zotero.DB.valueQueryAsync("SELECT COUNT(*) FROM libraries WHERE version=0"))) {
			await Zotero.DB.queryAsync(
				"DELETE FROM version WHERE schema IN ('lastlocalsync', 'lastremotesync')"
			);
		}
	}.bind(this));
	
	Zotero.debug("Done upgrading " + this.library.name);
	
	return versionResults;
};


/**
 * Perform a full sync
 *
 * Get all object versions from the API and compare to the local database. If any objects are
 * missing or outdated, download them. If any local objects are marked as synced but aren't available
 * remotely, mark them as unsynced for later uploading.
 *
 * (Technically this isn't a full sync on its own, because local objects are only flagged for later
 * upload.)
 *
 * @param {Object[]} [versionResults] - Objects returned from getVersions(), keyed by objectType
 * @return {Promise<Integer>} - Promise for the library version after syncing
 */
Zotero.Sync.Data.Engine.prototype._fullSync = async function (versionResults) {
	Zotero.debug("Performing a full sync of " + this.library.name);
	
	var gen;
	var lastLibraryVersion;
	
	loop:
	while (true) {
		this._statusCheck();
		
		// Reprocess all deletions available from API
		let results = await this._downloadDeletions(0);
		lastLibraryVersion = results.libraryVersion;
		
		// Get synced settings
		results = await this._downloadSettings(0, lastLibraryVersion);
		if (results.result == this.DOWNLOAD_RESULT_RESTART) {
			await this._onLibraryVersionChange();
			continue loop;
		}
		else {
			lastLibraryVersion = results.libraryVersion;
		}
		
		// Get object types
		for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) {
			this._statusCheck();
			
			let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
			let ObjectType = Zotero.Utilities.capitalize(objectType);
			let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
			
			// TODO: localize
			this.setStatus("Updating " + objectTypePlural + " in " + this.library.name);
			
			let results = {};
			// Use provided versions
			if (versionResults) {
				results = versionResults[objectType];
			}
			// If not available, get from API
			else {
				results = await this.apiClient.getVersions(
					this.library.libraryType,
					this.libraryTypeID,
					objectType
				);
			}
			if (lastLibraryVersion != results.libraryVersion) {
				await this._onLibraryVersionChange();
				continue loop;
			}
			
			let toDownload = [];
			let localVersions = await objectsClass.getObjectVersions(this.libraryID);
			// Queue objects that are out of date or don't exist locally
			for (let key in results.versions) {
				let version = results.versions[key];
				let obj = objectsClass.getByLibraryAndKey(this.libraryID, key);
				// If object is already at or above latest version, skip. Local version can be
				// higher because, as explained in _uploadObjects(), we upload items in batches
				// and only get the last version to record in the database.
				let localVersion = localVersions[key];
				if (localVersion && localVersion >= version) {
					continue;
				}
				
				if (obj) {
					Zotero.debug(`${ObjectType} ${obj.libraryKey} is older than remote version`);
				}
				else {
					Zotero.debug(`${ObjectType} ${this.libraryID}/${key} does not exist locally`);
				}
				
				toDownload.push(key);
			}
			
			if (toDownload.length) {
				Zotero.debug("Downloading missing/outdated " + objectTypePlural);
				await this._downloadObjects(objectType, toDownload);
			}
			else {
				Zotero.debug(`No missing/outdated ${objectTypePlural} to download`);
			}
			
			// Mark local objects that don't exist remotely as unsynced and version 0
			let allKeys = await objectsClass.getAllKeys(this.libraryID);
			let remoteMissing = Zotero.Utilities.arrayDiff(allKeys, Object.keys(results.versions));
			if (remoteMissing.length) {
				Zotero.debug("Checking remotely missing " + objectTypePlural);
				Zotero.debug(remoteMissing);
				
				let toUpload = remoteMissing.map(
					key => objectsClass.getIDFromLibraryAndKey(this.libraryID, key)
				// Remove any objects deleted since getAllKeys() call
				).filter(id => id);
				
				// For remotely missing objects that exist locally, reset version, since old
				// version will no longer match remote, and mark for upload
				if (toUpload.length) {
					Zotero.debug(`Marking remotely missing ${objectTypePlural} as unsynced`);
					await objectsClass.updateVersion(toUpload, 0);
					await objectsClass.updateSynced(toUpload, false);
				}
			}
			else {
				Zotero.debug(`No remotely missing synced ${objectTypePlural}`);
			}
		}
		break;
	}
	
	this.library.libraryVersion = lastLibraryVersion;
	await this.library.saveTx();
	
	Zotero.debug("Done with full sync for " + this.library.name);
	
	return lastLibraryVersion;
};


Zotero.Sync.Data.Engine.prototype._restoreToServer = async function () {
	Zotero.debug("Performing a restore-to-server for " + this.library.name);
	
	var libraryVersion;
	
	// Flag engine as restore-to-server mode so it uses library version only
	this._restoringToServer = true;
	
	await Zotero.DB.executeTransaction(async function () {
		await Zotero.Sync.Data.Local.clearCacheForLibrary(this.libraryID);
		await Zotero.Sync.Data.Local.clearQueueForLibrary(this.libraryID);
		await Zotero.Sync.Data.Local.clearDeleteLogForLibrary(this.libraryID);
		
		// Mark all local settings as unsynced
		await Zotero.SyncedSettings.markAllAsUnsynced(this.libraryID);
		
		// Mark all objects as unsynced
		for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) {
			let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
			// Reset version on all objects and mark as unsynced
			let ids = await objectsClass.getAllIDs(this.libraryID)
			await objectsClass.updateVersion(ids, 0);
			await objectsClass.updateSynced(ids, false);
		}
	}.bind(this));
	
	var remoteUpdatedError = "Online library updated since restore began";
	
	for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) {
		this._statusCheck();
		
		let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
		let ObjectType = Zotero.Utilities.capitalize(objectType);
		let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
		
		// Get all object versions from the API
		let results = await this.apiClient.getVersions(
			this.library.libraryType,
			this.libraryTypeID,
			objectType
		);
		if (libraryVersion && libraryVersion != results.libraryVersion) {
			throw new Error(remoteUpdatedError
				+ ` (${libraryVersion} != ${results.libraryVersion})`);
		}
		libraryVersion = results.libraryVersion;
		
		// Filter to objects that don't exist locally and delete those objects remotely
		let remoteKeys = Object.keys(results.versions);
		let locallyMissingKeys = remoteKeys.filter((key) => {
			return !objectsClass.getIDFromLibraryAndKey(this.libraryID, key);
		});
		if (locallyMissingKeys.length) {
			Zotero.debug(`Deleting remote ${objectTypePlural} that don't exist locally`);
			try {
				libraryVersion = await this._uploadDeletions(
					objectType, locallyMissingKeys, libraryVersion
				);
			}
			catch (e) {
				if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
					// Let's just hope this doesn't happen
					if (e.status == 412) {
						throw new Error(remoteUpdatedError);
					}
				}
				throw e;
			}
		}
		else {
			Zotero.debug(`No remote ${objectTypePlural} that don't exist locally`);
		}
	}
	
	this.library.libraryVersion = libraryVersion;
	await this.library.saveTx();
	
	// Upload the local data, which has all been marked as unsynced. We could just fall through to
	// the normal _startUpload() in start(), but we don't want to accidentally restart and
	// start downloading data if there's an error condition, so it's safer to call it explicitly
	// here.
	var uploadResult;
	try {
		uploadResult = await this._startUpload();
	}
	catch (e) {
		if (e instanceof Zotero.Sync.UserCancelledException) {
			throw e;
		}
		Zotero.logError("Restore-to-server failed for " + this.library.name);
		throw e;
	}
	
	Zotero.debug("Upload result is " + uploadResult, 4);
	
	switch (uploadResult) {
	case this.UPLOAD_RESULT_SUCCESS:
	case this.UPLOAD_RESULT_NOTHING_TO_UPLOAD:
		// Force all files to be checked for upload. If an attachment's hash was changed, it will
		// no longer have an associated file, and then upload check will cause a file to be
		// uploaded (or, more likely if this is a restoration from a backup, reassociated with
		// another existing file). If the attachment's hash wasn't changed, it should already
		// have the correct file.
		await Zotero.Sync.Storage.Local.resetAllSyncStates(this.libraryID);
		
		Zotero.debug("Restore-to-server completed");
		break;
	
	case this.UPLOAD_RESULT_LIBRARY_CONFLICT:
		throw new Error(remoteUpdatedError);
	
	case this.UPLOAD_RESULT_RESTART:
		return this._restoreToServer()
	
	case this.UPLOAD_RESULT_CANCEL:
		throw new Zotero.Sync.UserCancelledException;
	
	default:
		throw new Error("Restore-to-server failed for " + this.library.name);
	}
	
	this._restoringToServer = false;
};


Zotero.Sync.Data.Engine.prototype._getOptions = function (additionalOpts = {}) {
	var options = {};
	this.optionNames.forEach(x => options[x] = this[x]);
	for (let opt in additionalOpts) {
		options[opt] = additionalOpts[opt];
	}
	return options;
}


Zotero.Sync.Data.Engine.prototype._handleUploadError = async function (e) {
	if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
		switch (e.status) {
		// This should only happen if library permissions were changed between the group check at
		// sync start and now, or to people who upgraded from <5.0-beta.r25+66ca2cf with unsynced local
		// changes.
		case 403:
			let index = Zotero.Sync.Data.Utilities.showWriteAccessLostPrompt(null, this.library);
			if (index === 0) {
				await Zotero.Sync.Data.Local.resetUnsyncedLibraryData(this.libraryID);
				return this.UPLOAD_RESULT_RESTART;
			}
			if (index == 1) {
				return this.UPLOAD_RESULT_CANCEL;
			}
			throw new Error(`Unexpected index value ${index}`);
		
		case 409: // TEMP: from classic sync
		case 412:
			return this.UPLOAD_RESULT_LIBRARY_CONFLICT;
		}
	}
	else if (e.name == "ZoteroObjectUploadError") {
		switch (e.code) {
		case 404:
		case 412:
			return this.UPLOAD_RESULT_OBJECT_CONFLICT;
		}
	}
	else if (e.name == "ZoteroUploadRestartError") {
		return this.UPLOAD_RESULT_RESTART;
	}
	else if (e.name == "ZoteroUploadCancelError") {
		return this.UPLOAD_RESULT_CANCEL;
	}
	throw e;
};


Zotero.Sync.Data.Engine.prototype._checkObjectUploadError = async function (objectType, key, e, queue, batch) {
	var { code, data, message } = e;
	
	// If an item's dependency is missing remotely and it isn't in the queue (which
	// shouldn't happen), mark it as unsynced
	if (code == 400 || code == 409) {
		if (data) {
			if (objectType == 'collection' && code == 409) {
				if (data.collection) {
					let collection = Zotero.Collections.getByLibraryAndKey(this.libraryID, data.collection);
					if (!collection) {
						throw new Error(`Collection ${this.libraryID}/${key} `
							+ `references parent collection ${data.collection}, which doesn't exist`);
					}
					Zotero.logError(`Marking collection ${data.collection} as unsynced`);
					await Zotero.Sync.Data.Local.markObjectAsUnsynced(collection);
				}
			}
			else if (objectType == 'item') {
				if (data.collection) {
					let collection = Zotero.Collections.getByLibraryAndKey(this.libraryID, data.collection);
					if (!collection) {
						throw new Error(`Item ${this.libraryID}/${key} `
							+ `references collection ${data.collection}, which doesn't exist`);
					}
					Zotero.logError(`Marking collection ${data.collection} as unsynced`);
					await Zotero.Sync.Data.Local.markObjectAsUnsynced(collection);
				}
				else if (data.parentItem) {
					let parentItem = Zotero.Items.getByLibraryAndKey(this.libraryID, data.parentItem);
					if (!parentItem) {
						throw new Error(`Item ${this.libraryID}/${key} references parent `
							+ `item ${data.parentItem}, which doesn't exist`);
					}
					
					let id = parentItem.id;
					// If parent item isn't already in queue, mark it as unsynced and add it
					if (!queue.find(o => o.id == id)
							// TODO: Don't use 'delete' on batch, which results in undefineds
							&& !batch.find(o => o && o.id == id)) {
						await Zotero.Sync.Data.Local.markObjectAsUnsynced(parentItem);
						Zotero.logError(`Adding parent item ${data.parentItem} to upload queue`);
						queue.push({
							id,
							json: null,
							tries: 0,
							failed: false
						});
						// Pretend that we were successful so syncing continues
						return true;
					}
				}
			}
		}
		
		// Split old annotations before we were splitting them automatically in the PDF reader
		if (message.startsWith("Annotation position is too long")) {
			let item = Zotero.Items.getByLibraryAndKey(this.libraryID, key);
			await Zotero.Annotations.splitAnnotations(item);
		}
	}
	else if (code == 403) {
		// If we get a 403 for a local group attachment, check the group permissions to confirm
		// that we no longer have file-editing access and prompt to reset local group files
		if (objectType == 'item') {
			let item = Zotero.Items.getByLibraryAndKey(this.libraryID, key);
			if (this.library.libraryType == 'group' && item.isFileAttachment()) {
				let reset = false;
				let groupID = Zotero.Groups.getGroupIDFromLibraryID(this.libraryID);
				let info = await this.apiClient.getGroup(groupID);
				if (info) {
					Zotero.debug(info);
					let { editable, filesEditable } = Zotero.Groups.getPermissionsFromJSON(
						info.data, this.userID
					);
					// If we do still have file-editing access, something else went wrong,
					// and we should just fail without resetting
					if (!filesEditable) {
						let index = Zotero.Sync.Storage.Utilities.showFileWriteAccessLostPrompt(
							null, this.library
						);
						
						let e = new Error(message);
						if (index === 0) {
							let group = Zotero.Groups.get(groupID);
							group.filesEditable = false;
							await group.saveTx();
							
							await Zotero.Sync.Data.Local.resetUnsyncedLibraryFiles(this.libraryID);
							e.name = "ZoteroUploadRestartError";
						}
						else {
							e.name = "ZoteroUploadCancelError";
						}
						throw e;
					}
				}
				else {
					Zotero.logError("Couldn't get metadata for group " + groupID);
				}
			}
		}
	}
	// This shouldn't happen, because the upload request includes a library version and should
	// prevent an outdated upload before the object version is checked. If it does, we need to
	// do a full sync. This error is checked in handleUploadError().
	else if (code == 404 || code == 412) {
		throw e;
	}
	
	return false;
};


Zotero.Sync.Data.Engine.prototype._statusCheck = function () {
	this._stopCheck();
	this._failedCheck();
}


Zotero.Sync.Data.Engine.prototype._stopCheck = function () {
	if (!this._stopping) return;
	Zotero.debug("Sync stopped for " + this.library.name);
	throw new Zotero.Sync.UserCancelledException;
}


Zotero.Sync.Data.Engine.prototype._failedCheck = function () {
	if (this.stopOnError && this.failed) {
		Zotero.logError("Stopping on error");
		throw this.failed;
	}
};
