Node.js script for copying files between two FTP locations
New here? Learn about Bountify and follow @bountify to get notified of new bounties! x

I need a some javascript to copy new files from a source FTP location to a destination FTP location.
Both FTP locations are on different servers and both are secured by username and password.

New files on the source FTP can be identified by the following pattern "directory/subdirectory/filename*.tar.gz".
Any files matching this pattern must be copied to the destination location.

For example:

"directory/subdirectory/filename123.tar.gz" --> must be copied

"directory/subdirectory/filenameABC.tar.gz" --> must be copied

"directory/filename123.tar.gz" --> must not be copied

"directory/subdirectory/123.tar.gz" --> must not be copied

"directory/subdirectory/filename123.txt" --> must not be copied

Ideally it's a node js script running server side with some standard npm package for ftp features.

awarded to kostasx

Crowdsource coding tasks.

2 Solutions


First:

$ npm install ftp

Copy the following code into a file ftpc.js and:

$ node ftpc.js

var fs     = require('fs');
var Client = require('ftp');

// SOURCE FTP CONNECTION SETTINGS
var srcFTP = {    

    host     : 'ftp.drivehq.com',
    port     : 21,
    user     : 'usename',
    password : 'password'

}

// DESTINATION FTP CONNECTION SETTINGS
var destFTP = {

    host     : 'ftp.drivehq.com',
    port     : 21,
    user     : 'username',
    password : 'password'

}

var downloadList = [];

var c = new Client();
c.on('ready', function() {

  c.list( 'directory/subdirectory/', function(err, list) {

    if (err) throw err;

    list.map(function(entry){
      if ( entry.name.match(/tar\.gz$/) && entry.name.match(/^filename/) ){
        downloadList.push(entry.name);
      }
    });

    downloadList.map(function(file){

      // Download remote files and save it to the local file system:
      c.get('directory/subdirectory/' + file, function(err, stream) {

        if (err) throw err;
        stream.once('close', function() { c.end(); });
        stream.pipe(fs.createWriteStream(file));

      });

    });

    c.end();

  });

});

c.on('end', function(){

  if ( downloadList.length > 0 ){

    console.log("Uploading...");

    var d = new Client();
    d.on('ready', function() {

      downloadList.map(function(filename){

        // Upload local files to the server:
        d.put(filename, filename, function(err) {
          if (err) throw err;
          d.end();
        });

      });

    });

    d.connect(destFTP);


  } else {

    console.log("Error: Download list empty.");

  }

});

c.connect(srcFTP);
This code should work but you've added extra step of saving files locally before pushing them on the destination FTP. I want to avoid this step...any idea ?
ddjembi 11 months ago
ddjemi, both your FTP servers would need a functionality known as FXP for that to be possible. If you have complete access to one of the servers (let this server be the "destination" server) then you can set that one up to just download from the sourrce FTP into a directory being served through the FTP server on that machine. In that case you remove the upload part of this script and run it on the destination machine.
MrJerB 11 months ago
MrJeB, your solution deviates too much from the specs. Please assume I have no control on any of the servers, just read access on the source FTP and read/write on the destination FTP.
ddjembi 11 months ago
@ddjemi. What if we limit saving to only one temporary file locally that will be removed upon completion?
kostasx 11 months ago
@ddjembi I never posted a solution, only a comment regarding the additional requirement you would need to transfer FTP server-to-server.
MrJerB 11 months ago
Winning solution

Here is a solution, that uses the Transform Streams to pass data from source FTP to destination FTP, without the need to store files locally.

var fs     = require('fs');
var Client = require('ftp');

// SOURCE FTP CONNECTION SETTINGS
var srcFTP = {    

    host     : 'ftp.drivehq.com',
    port     : 21,
    user     : 'username',
    password : 'password'

}

// DESTINATION FTP CONNECTION SETTINGS
var destFTP = {

    host     : 'ftp.drivehq.com',
    port     : 21,
    user     : 'username',
    password : 'password'

}

var downloadList = [];
var Transform = require('stream').Transform;

function passThrough() {
    var passthrough = new Transform();
    passthrough._transform = function(data, encoding, done) {
        console.log(data);
        this.push(data);
        done();
    };
    return passthrough;
}

var d = new Client();
d.on('ready', function() {

  var c = new Client();
  c.on('ready', function() {

    c.list( 'directory/subdirectory/', function(err, list) {

      if (err) throw err;

      list.map(function(entry){
        if ( entry.name.match(/tar\.gz$/) && entry.name.match(/^filename/) ){
          downloadList.push(entry.name);
        }
      });

      downloadList.map(function(file,index){

        // Download remote files and save it to the local file system:
        c.get('directory/subdirectory/' + file, function(err, stream) {

          if (err) throw err;

          // Upload local files to the server:
          d.put(stream.pipe(passThrough()), file, function(err) {
            if ( err ) throw err;
            if ( downloadList.length === index + 1 ) c.end();
          });

        });

      });

    });

  });

  c.on('end', function(){
    d.end();   
  });

  c.connect(srcFTP);

});

d.connect(destFTP);
@kostasx, looks fine...I'll test it this evening. Tx
ddjembi 11 months ago
@kostasx, tested and working like a charm. Thanks !
ddjembi 11 months ago
Thank you ddjembi!
kostasx 11 months ago
View Timeline