Browser based editing of files on Dexter via node web server

[JamesNewton]

Updating / editing files on Dexter can be difficult without the support of Samba, which is not supported in recent versions of WIndows ( see issue #58 ). Of course, we can SSH in, but many users are not experienced with this method and the nano or vim editors leave much to be desired.

The Ace editor is very capable and quite compact, at only 354kb for a full featured editor, with all the standard features, and code editing, parentheses highlighting, syntax checking (lint) for C, C++, JavaScript, CSS, and HTML among others.

With this editor available on Dexter via a standard browser, anyone can edit firmware, job files, settings files (.make_ins), as well as scripts, etc...

To install this on Dexter, just do the batch install of the node server, which contains updated versions of all the required files.

To build from scratch, first the node web engine is required. See Node.js web server

1. https://github.com/node-formidable/formidable
   is required to process the POST data coming back when a file is saved. It must be installed on Dexter while the robot is connected to the internet, via
   npm install formidable from the /srv/samba/share folder.
2. a new folder called "edit" needs to be created under /srv/samba/share/www.
3. In /srv/samba/share/edit, the files "edit.html", "page.png", "folder.png", and "ace.js" must be added. Several other files are required for syntax highlighting and search. All the files are available in the node server batch update .zip file
   Note: Credit for the edit.html file (which some modifications) should be given to:
   https://github.com/me-no-dev/ESPAsyncWebServer/blob/master/src/edit.htm
4. In /srv/samba/share/www/index.html a link to the edit function should be added. (also in the web batch)
5. the /srv/samba/share/www/httpd.js file must be edited (as it is in the web batch) to add
   const formidable = require('formidable')
   near the start and to replace the standard web server section with:

function isBinary(byte) { //must use numbers, not strings to compare. ' ' is 32
  if (byte >= 32 && byte < 128) {return false} //between space and ~
  if ([13, 10, 9].includes(byte)) { return false } //or text ctrl chars
  return true
}

//standard web server on port 80 to serve files
var http_server = http.createServer(function (req, res) {
  //see https://nodejs.org/api/http.html#http_class_http_incomingmessage 
  //for the format of q. 
  var q = url.parse(req.url, true)
  console.log("web server passed pathname: " + q.pathname)
  if (q.pathname === "/") {
      q.pathname = "index.html"
  }
  if (q.pathname === "/init_jobs") {
      serve_init_jobs(q, req, res)
  }
  else if (q.pathname === "/edit" && q.query.list ) { 
    let path = SHARE_FOLDER + q.query.list
    console.log("File list:"+path)
    fs.readdir(path, {withFileTypes: true}, 
      function(err, items){ //console.log("file:" + JSON.stringify(items))
        let dir = []
        if (q.query.list != "/") { //not at root
          dir.push({name: "..", size: "", type: "dir"})
          }
        for (i in items) { //console.log("file:", JSON.stringify(items[i]))
          if (items[i].isFile()) { 
            let stats = fs.statSync(path + items[i].name)
            let size = stats["size"]
            dir.push({name: items[i].name, size: size, type: "file"})
            } //size is never actually used.
          else if (items[i].isDirectory()) {
            dir.push({name: items[i].name, size: "", type: "dir"})
            } //directories are not currently supported. 
          }
        res.write(JSON.stringify(dir))
        res.end()
      })
    }
  else if (q.pathname === "/edit" && q.query.edit ) { 
    let filename = SHARE_FOLDER + q.query.edit
    console.log("serving" + filename)
    fs.readFile(filename, function(err, data) {
        if (err) {
            res.writeHead(404, {'Content-Type': 'text/html'})
            return res.end("404 Not Found")
        }
        let stats = fs.statSync(filename)
        console.log(("permissions:" + (stats.mode & parseInt('777', 8)).toString(8)))
        for (let i = 0; i < data.length; i++) { 
          if ( isBinary(data[i]) ) { console.log("binary data:" + data[i] + " at:" + i)
            res.setHeader("Content-Type", "application/octet-stream")
            break
            }
          }
        res.writeHead(200)
        res.write(data)
        return res.end()
      })
    }
    else if (q.pathname === "/edit" && req.method == 'POST' ) { //console.log("edit post file")
        const form = formidable({ multiples: false });
        form.once('error', console.error);
        const DEFAULT_PERMISSIONS = parseInt('644', 8)
        var stats = {mode: DEFAULT_PERMISSIONS}
        form.on('file', function (filename, file) { 
          try { console.log("copy", file.path, "to", SHARE_FOLDER + file.name)
            stats = fs.statSync(SHARE_FOLDER + file.name) 
            console.log(("had permissions:" + (stats.mode & parseInt('777', 8)).toString(8)))
          } catch {} //no biggy if that didn't work
          fs.copyFile(file.path, SHARE_FOLDER + file.name, function(err) {
            let new_mode = undefined
            if (err) { console.log("copy failed:", err)
              res.writeHead(400)
              return res.end("Failed")
              }
            else {
              fs.chmodSync(SHARE_FOLDER + file.name, stats.mode)
              try { //sync ok because we will recheck the actual file
                let new_stats = fs.statSync(SHARE_FOLDER + file.name)
                new_mode = new_stats.mode
                console.log(("has permissions:" + (new_mode & parseInt('777', 8)).toString(8)))
              } catch {} //if it fails, new_mode will still be undefined
              if (stats.mode != new_mode) { //console.log("permssions wrong")
                //res.writeHead(400) //no point?
                return res.end("Permissions error")
                }
              fs.unlink(file.path, function(err) {
                if (err) console.log(file.path, 'not cleaned up', err);
                }); 
              res.end('ok');
              }
            }) //done w/ copyFile
          });
        form.parse(req)
        //res.end('ok');
      // });
      }
      else if (q.pathname === "/edit" && req.method == 'PUT' ) { console.log('edit put')
        const form = formidable({ multiples: true });
        form.parse(req, (err, fields, files) => { //console.log('fields:', fields);
          let pathfile = SHARE_FOLDER + fields.path
          fs.writeFile(pathfile, "", function (err) { console.log('create' + pathfile)
            if (err) {console.log("failed", err)
              res.writeHead(400)
              return res.end("Failed:" + err)
              }
           res.end('ok'); //console.log('done');
           }); 
          });
        }
      //else if(q.pathname === "/job_button_click") {
  //	  serve_job_button_click(q, req, res)
  //}
  //else if(q.pathname === "/show_window_button_click") {
  //	  serve_show_window_button_click(q, req, res)
  //} 
  else {
  	  serve_file(q, req, res)
  }
})

Note: The version in the web batch also has the changes required to better support running job engine jobs with a full 2 way interface allowing show_dialog calls to appear in the browser.

DONE: Testing. Not sure if it screws up executable flags. May need to check and chmod / chown after writing new files.
DONE: Add support for changing directories.
DONE: Figure out what to do about binary / very large files. The editor now allows you to edit all files, but warns if the file is very large or if binary codes are detected in the file.
DONE: Create new files.
DONE: Upload files.
DONE: Delete files? Or just move to /tmp folder

[JamesNewton]

TODO:

• Update Dexters clock from the PC if Dexter is stuck in the 1970's (no real time clock on board, and without access to NTP, when connected directly to a PC, it has no idea what the date / time is)
• Chmod files? e.g. toggle executable. Would be nice for creating new scripts. Right now, the permissions are retained from an existing file, but not for a new one.
• Find some way to recompile DexRun.c to DexRun via a button in the editor.