diff --git a/package.json b/package.json
index 3c1432e3d999b18a1c66fc6debd265aead2aab51..53b1aa4694828dac6109ab474147565a58243d54 100644
--- a/package.json
+++ b/package.json
@@ -24,6 +24,7 @@
"nunjucks": "^2.3.0",
"nunjucks-date-filter": "^0.1.1",
"sqlite-parser": "^0.14.3",
+ "ssh2": "^0.5.0",
"syntax-error": "^1.1.5",
"toml": "^2.3.0",
"toml-js": "0.0.8",
diff --git a/sftp-sync.js b/sftp-sync.js
new file mode 100644
index 0000000000000000000000000000000000000000..69771a130059af08bef64a4d61edf0a2d997ba6a
--- /dev/null
+++ b/sftp-sync.js
@@ -0,0 +1,302 @@
+var crypto = require('crypto');
+var walk = require('walk');
+var fs = require('fs');
+var path = require('path');
+var cli = require('cli');
+
+var options = cli.parse({
+ sourcedir: [null, 'From where to upload (required)', 'string'],
+ remotedir: [null, 'Target directory for upload (required)', 'string'],
+ server: ['s', 'SFTP server (required)', 'string'],
+ user: ['u', 'SFTP user (required)', 'string'],
+ port: ['p', 'port', 'number', 22]
+});
+
+function missingArguments() { console.log('Missing required arguments. Use --help.')}
+if(!options.sourcedir) return missingArguments();
+if(!options.server) return missingArguments();
+if(!options.user) return missingArguments();
+if(!options.remotedir) return missingArguments();
+if(!process.env.PASSWORD) return console.log('Missing environment variable PASSWORD');
+
+var Client = require('ssh2').Client;
+
+var conn = new Client();
+function connect(arg) {
+ return new Promise(function(resolve, reject) {
+ var connReady = false;
+
+ var resolved = false;
+ conn.once('ready', function() {
+ console.log('Client :: ready');
+ if(!resolved) {
+ resolved = true;
+ resolve(arg);
+ }
+ })
+ .once('error', function() {
+ if(resolved) {
+ console.log(new Error('Failed to connect to server').stack);
+ process.exit(0);
+ } else {
+ resolved = true;
+ reject(new Error('Failed to connect to server'));
+ }
+ })
+ .connect({
+ host: options.server,
+ port: options.port,
+ username: options.user,
+ password: process.env.PASSWORD
+ });
+ })
+}
+
+function checksum (str, algorithm, encoding) {
+ return crypto
+ .createHash(algorithm || 'sha256')
+ .update(str, 'utf8')
+ .digest(encoding || 'hex')
+}
+
+function fileChecksum(file) {
+ return new Promise(function(resolve, reject) {
+ fs.readFile(file, function (err, data) {
+ if(err) reject(new Error('Error reading file '+ file+' '+err));
+ resolve({ file: file, checksum: checksum(data)});
+ });
+ })
+}
+
+new Promise(function(resolve, reject) {
+ console.log('Upload step: Listing files');
+ var files = [];
+ walk.walk(options.sourcedir)
+ .on('file',function(root,fileStats,next) {
+ var filename = path.join(root,fileStats.name);
+ files.push(filename);
+ next();
+ })
+ .on('errors',function(root, nodeStatsArray, next) {
+ console.log('Walker error', root, nodeStatsArray);
+ next();
+ })
+ .on('end', function() {
+ resolve(files);
+ })
+})
+.then(function(files) {
+ console.log('Upload step: Generating checksums');
+ var promises = [];
+ files.forEach(function(file) {
+ promises.push(fileChecksum(file))
+ })
+ return Promise.all(promises);
+})
+.then(function(localchecksums) {
+ console.log('Upload step: Making paths relative')
+ var r = [];
+ localchecksums.forEach(function(cs) {
+ cs.file = path.relative(options.sourcedir, cs.file);
+ r.push(cs);
+ })
+ return r;
+})
+.then(connect)
+.then(function(localchecksums) {
+ console.log('Upload step: Getting remote checksums');
+ return new Promise(function(resolve, reject) {
+ conn.sftp(function(err, sftp) {
+ if (err) reject(err);
+ var data = '';
+ sftp.createReadStream(options.remotedir+'/checksums.json', 'utf-8')
+ .on('data', function(chunk) {
+ data+=chunk;
+ })
+ .on('end', function() {
+ resolve({local: localchecksums, remote: JSON.parse(data), sftp: sftp});
+ })
+ .on('error', function(e) {
+ resolve({local: localchecksums, remote: [], sftp: sftp});
+ })
+ });
+ })
+})
+/*
+.then(function(obj) {
+ console.log('Upload step: Listing remote directory');
+
+ return new Promise(function(resolve, reject) {
+ var files = [];
+ var readDir = function(dir) {
+ console.log('Reading dir', dir);
+ return new Promise(function(resolve, reject) {
+ obj.sftp.readdir(dir, function(err, list) {
+ if (err) reject(err);
+ var promises = [];
+ list.forEach(function(entry) {
+ if(entry.longname.substring(0,1) == 'd') { // directory
+ promises.push(readDir(dir+'/'+entry.filename));
+ } else {
+ promises.push(Promise.resolve(dir+'/'+entry.filename));
+ }
+ })
+ resolve(Promise.all(promises));
+ });
+ });
+ }
+ var flatten = function(array) {
+ return [].concat.apply([], array);
+ }
+ readDir(options.remotedir)
+ .then(flatten)
+ .then(function(list) {
+ obj.remoteFiles = list;
+ resolve(obj);
+ })
+ });
+})
+*/
+.then(function(obj) {
+ console.log('Upload step: Deleting old files');
+
+ var localAssoc = {};
+ obj.local.forEach(function(entry) {
+ localAssoc[entry.file] = entry.checksum;
+ })
+
+ var promises = [];
+
+ obj.remote.forEach(function(entry) {
+ if(localAssoc[entry.file] != entry.checksum && entry.file != 'checksums.json') {
+ console.log(' deleting',entry.file);
+ promises.push(new Promise(function(resolve, reject) {
+ obj.sftp.unlink(options.remotedir+'/'+entry.file, function(err) {
+ if(err) console.log(' [Warning] Error deleting '+entry.file);
+ resolve();
+ })
+ }));
+ }
+ })
+
+ return Promise.all(promises)
+ .then(function(){
+ return obj;
+ })
+})
+.then(function(obj) {
+ console.log('Upload step: Creating required directories');
+
+ var remoteAssoc = {};
+ obj.remote.forEach(function(entry) {
+ remoteAssoc[entry.file] = entry.checksum;
+ })
+
+ var toUpload = [];
+ obj.local.forEach(function(entry) {
+ if(remoteAssoc[entry.file] != entry.checksum) {
+ var dir = path.dirname(entry.file);
+ if(dir !== '.') toUpload.push(dir);
+ }
+ })
+ function uniq(a) {
+ var seen = {};
+ return a.filter(function(item) {
+ return seen.hasOwnProperty(item) ? false : (seen[item] = true);
+ });
+ }
+ toUpload = uniq(toUpload);
+ var pushParents = function(dir) {
+ var parent = path.dirname(dir);
+ if(parent == '.') return;
+ toUpload.push(parent);
+ pushParents(parent);
+ }
+ toUpload.forEach(function(file) {
+ pushParents(file);
+ })
+ toUpload = uniq(toUpload.reverse());
+
+ var promises = [];
+ toUpload.forEach(function(dir) {
+ promises.push(new Promise(function(resolve, reject) {
+ console.log('Creating directory '+dir);
+ obj.sftp.mkdir(options.remotedir+'/'+dir,function() {
+ resolve();
+ })
+ }))
+ })
+ return Promise.all(promises)
+ .then(function() {
+ return obj;
+ })
+})
+.then(function(obj) {
+ console.log('Upload step: Uploading new files');
+
+ var remoteAssoc = {};
+ obj.remote.forEach(function(entry) {
+ remoteAssoc[entry.file] = entry.checksum;
+ })
+
+ var promises = [];
+
+ obj.local.forEach(function(entry) {
+ if(remoteAssoc[entry.file] != entry.checksum) {
+ console.log(' Uploading',entry.file);
+ promises.push(new Promise(function(resolve, reject) {
+ obj.sftp.fastPut(options.sourcedir+'/'+entry.file,options.remotedir+'/'+entry.file, function(err) {
+ if(err)console.log(' [Warning] Error uploading '+entry.file);
+ else console.log(' Uploaded',entry.file);
+ resolve();
+ })
+ }));
+ }
+ })
+
+ return Promise.all(promises)
+ .then(function(){
+ return obj;
+ })
+})
+.then(function(obj) {
+ console.log('Upload step: Uploading checksums.json');
+ return new Promise(function(resolve, reject) {
+ console.log(' Creating checksums.json');
+ fs.writeFile(options.sourcedir+'/checksums.json', JSON.stringify(obj.local, null, ' '), 'utf-8', function() {
+ resolve();
+ });
+ })
+ .then(function() {
+ console.log(' Removing remote checksums.json');
+ return new Promise(function(resolve, reject) {
+ obj.sftp.unlink(options.remotedir+'/checksums.json', function(err) {
+ resolve();
+ })
+ })
+ })
+ .then(function() {
+ console.log(' Uploading checksums.json');
+ return new Promise(function(resolve, reject){
+ obj.sftp.fastPut(options.sourcedir+'/checksums.json',options.remotedir+'/checksums.json', function(err) {
+ if(err)console.log(' [Warning] Error uploading checksums.json');
+ resolve();
+ })
+ });
+ })
+ .then(function() {
+ console.log(' Removing local checksums.json');
+ return new Promise(function(resolve, reject){
+ fs.unlink(options.sourcedir+'/checksums.json', function() {
+ resolve();
+ })
+ })
+ })
+})
+.then(function() {
+ conn.end();
+})
+.catch(function(e) {
+ conn.end();
+ console.log(e.stack);
+})
diff --git a/upload b/upload
index ed8bc877120a82207da4338ad45d9d1884c010cc..3a7c012f6fccdd5105ed898426b6f4cbbefac5fd 100755
--- a/upload
+++ b/upload
@@ -4,6 +4,7 @@ then
echo -n "Heslo pro upload: "
read -s LFTP_PASSWORD; export LFTP_PASSWORD
fi
+export PASSWORD=$LFTP_PASSWORD
USER=ok1kvk.cz-www-nove
HOST=krios.blueboard.cz
@@ -26,16 +27,7 @@ if [ "$1" == "ftp" ]; then
echo "LFTP finished with return code $RET"
else
echo "Using SFTP"
- time lftp -e "set sftp:auto-confirm yes;\
- set cmd:fail-exit yes;\
- set net:timeout 5;\
- set net:reconnect-interval-base $RECONNECT_INTERVAL;\
- set net:max-retries $MAX_RETRIES;\
- open --user $USER --env-password -p 2121 sftp://$HOST/;\
- mirror -c --verbose=9 -e -R -L ./build /;\
- exit 0;"
- RET=$?
- echo "LFTP finished with return code $RET"
+ /usr/bin/time -f "Upload took %e" -- node sftp-sync.js --sourcedir build --server $HOST --user $USER --remotedir /test --port 2121
fi
exit 0