Commit 6a3bac13 authored by Isabella Skořepová's avatar Isabella Skořepová

Custom sftp upload script

parent 6b1e96b3
Pipeline #200 passed with stages
var crypto = require('crypto');
var walk = require('walk');
var fs = require('fs');
var path = require('path');
var cli = require('cli');
var options = cli.parse({
sourcedir: [null, 'From where to upload (required)', 'string'],
remotedir: [null, 'Target directory for upload (required)', 'string'],
server: ['s', 'SFTP server (required)', 'string'],
user: ['u', 'SFTP user (required)', 'string'],
port: ['p', 'port', 'number', 22]
});
function missingArguments() { console.log('Missing required arguments. Use --help.')}
if(!options.sourcedir) return missingArguments();
if(!options.server) return missingArguments();
if(!options.user) return missingArguments();
if(!options.remotedir) return missingArguments();
if(!process.env.PASSWORD) return console.log('Missing environment variable PASSWORD');
var Client = require('ssh2').Client;
var conn = new Client();
function connect(arg) {
return new Promise(function(resolve, reject) {
var connReady = false;
var resolved = false;
conn.once('ready', function() {
console.log('Client :: ready');
if(!resolved) {
resolved = true;
resolve(arg);
}
})
.once('error', function() {
if(resolved) {
console.log(new Error('Failed to connect to server').stack);
process.exit(0);
} else {
resolved = true;
reject(new Error('Failed to connect to server'));
}
})
.connect({
host: options.server,
port: options.port,
username: options.user,
password: process.env.PASSWORD
});
})
}
function checksum (str, algorithm, encoding) {
return crypto
.createHash(algorithm || 'sha256')
.update(str, 'utf8')
.digest(encoding || 'hex')
}
function fileChecksum(file) {
return new Promise(function(resolve, reject) {
fs.readFile(file, function (err, data) {
if(err) reject(new Error('Error reading file '+ file+' '+err));
resolve({ file: file, checksum: checksum(data)});
});
})
}
new Promise(function(resolve, reject) {
console.log('Upload step: Listing files');
var files = [];
walk.walk(options.sourcedir)
.on('file',function(root,fileStats,next) {
var filename = path.join(root,fileStats.name);
files.push(filename);
next();
})
.on('errors',function(root, nodeStatsArray, next) {
console.log('Walker error', root, nodeStatsArray);
next();
})
.on('end', function() {
resolve(files);
})
})
.then(function(files) {
console.log('Upload step: Generating checksums');
var promises = [];
files.forEach(function(file) {
promises.push(fileChecksum(file))
})
return Promise.all(promises);
})
.then(function(localchecksums) {
console.log('Upload step: Making paths relative')
var r = [];
localchecksums.forEach(function(cs) {
cs.file = path.relative(options.sourcedir, cs.file);
r.push(cs);
})
return r;
})
.then(connect)
.then(function(localchecksums) {
console.log('Upload step: Getting remote checksums');
return new Promise(function(resolve, reject) {
conn.sftp(function(err, sftp) {
if (err) reject(err);
var data = '';
sftp.createReadStream(options.remotedir+'/checksums.json', 'utf-8')
.on('data', function(chunk) {
data+=chunk;
})
.on('end', function() {
resolve({local: localchecksums, remote: JSON.parse(data), sftp: sftp});
})
.on('error', function(e) {
resolve({local: localchecksums, remote: [], sftp: sftp});
})
});
})
})
/*
.then(function(obj) {
console.log('Upload step: Listing remote directory');
return new Promise(function(resolve, reject) {
var files = [];
var readDir = function(dir) {
console.log('Reading dir', dir);
return new Promise(function(resolve, reject) {
obj.sftp.readdir(dir, function(err, list) {
if (err) reject(err);
var promises = [];
list.forEach(function(entry) {
if(entry.longname.substring(0,1) == 'd') { // directory
promises.push(readDir(dir+'/'+entry.filename));
} else {
promises.push(Promise.resolve(dir+'/'+entry.filename));
}
})
resolve(Promise.all(promises));
});
});
}
var flatten = function(array) {
return [].concat.apply([], array);
}
readDir(options.remotedir)
.then(flatten)
.then(function(list) {
obj.remoteFiles = list;
resolve(obj);
})
});
})
*/
.then(function(obj) {
console.log('Upload step: Deleting old files');
var localAssoc = {};
obj.local.forEach(function(entry) {
localAssoc[entry.file] = entry.checksum;
})
var promises = [];
obj.remote.forEach(function(entry) {
if(localAssoc[entry.file] != entry.checksum && entry.file != 'checksums.json') {
console.log(' deleting',entry.file);
promises.push(new Promise(function(resolve, reject) {
obj.sftp.unlink(options.remotedir+'/'+entry.file, function(err) {
if(err) console.log(' [Warning] Error deleting '+entry.file);
resolve();
})
}));
}
})
return Promise.all(promises)
.then(function(){
return obj;
})
})
.then(function(obj) {
console.log('Upload step: Creating required directories');
var remoteAssoc = {};
obj.remote.forEach(function(entry) {
remoteAssoc[entry.file] = entry.checksum;
})
var toUpload = [];
obj.local.forEach(function(entry) {
if(remoteAssoc[entry.file] != entry.checksum) {
var dir = path.dirname(entry.file);
if(dir !== '.') toUpload.push(dir);
}
})
function uniq(a) {
var seen = {};
return a.filter(function(item) {
return seen.hasOwnProperty(item) ? false : (seen[item] = true);
});
}
toUpload = uniq(toUpload);
var pushParents = function(dir) {
var parent = path.dirname(dir);
if(parent == '.') return;
toUpload.push(parent);
pushParents(parent);
}
toUpload.forEach(function(file) {
pushParents(file);
})
toUpload = uniq(toUpload.reverse());
var promises = [];
toUpload.forEach(function(dir) {
promises.push(new Promise(function(resolve, reject) {
console.log('Creating directory '+dir);
obj.sftp.mkdir(options.remotedir+'/'+dir,function() {
resolve();
})
}))
})
return Promise.all(promises)
.then(function() {
return obj;
})
})
.then(function(obj) {
console.log('Upload step: Uploading new files');
var remoteAssoc = {};
obj.remote.forEach(function(entry) {
remoteAssoc[entry.file] = entry.checksum;
})
var promises = [];
obj.local.forEach(function(entry) {
if(remoteAssoc[entry.file] != entry.checksum) {
console.log(' Uploading',entry.file);
promises.push(new Promise(function(resolve, reject) {
obj.sftp.fastPut(options.sourcedir+'/'+entry.file,options.remotedir+'/'+entry.file, function(err) {
if(err)console.log(' [Warning] Error uploading '+entry.file);
else console.log(' Uploaded',entry.file);
resolve();
})
}));
}
})
return Promise.all(promises)
.then(function(){
return obj;
})
})
.then(function(obj) {
console.log('Upload step: Uploading checksums.json');
return new Promise(function(resolve, reject) {
console.log(' Creating checksums.json');
fs.writeFile(options.sourcedir+'/checksums.json', JSON.stringify(obj.local, null, ' '), 'utf-8', function() {
resolve();
});
})
.then(function() {
console.log(' Removing remote checksums.json');
return new Promise(function(resolve, reject) {
obj.sftp.unlink(options.remotedir+'/checksums.json', function(err) {
resolve();
})
})
})
.then(function() {
console.log(' Uploading checksums.json');
return new Promise(function(resolve, reject){
obj.sftp.fastPut(options.sourcedir+'/checksums.json',options.remotedir+'/checksums.json', function(err) {
if(err)console.log(' [Warning] Error uploading checksums.json');
resolve();
})
});
})
.then(function() {
console.log(' Removing local checksums.json');
return new Promise(function(resolve, reject){
fs.unlink(options.sourcedir+'/checksums.json', function() {
resolve();
})
})
})
})
.then(function() {
conn.end();
})
.catch(function(e) {
conn.end();
console.log(e.stack);
})
......@@ -4,6 +4,7 @@ then
echo -n "Heslo pro upload: "
read -s LFTP_PASSWORD; export LFTP_PASSWORD
fi
export PASSWORD=$LFTP_PASSWORD
USER=ok1kvk.cz-www-nove
HOST=krios.blueboard.cz
......@@ -26,16 +27,7 @@ if [ "$1" == "ftp" ]; then
echo "LFTP finished with return code $RET"
else
echo "Using SFTP"
time lftp -e "set sftp:auto-confirm yes;\
set cmd:fail-exit yes;\
set net:timeout 5;\
set net:reconnect-interval-base $RECONNECT_INTERVAL;\
set net:max-retries $MAX_RETRIES;\
open --user $USER --env-password -p 2121 sftp://$HOST/;\
mirror -c --verbose=9 -e -R -L ./build /;\
exit 0;"
RET=$?
echo "LFTP finished with return code $RET"
/usr/bin/time -f "Upload took %e" -- node sftp-sync.js --sourcedir build --server $HOST --user $USER --remotedir /test --port 2121
fi
exit 0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment