1 // vim: set ts=4 sts=4 sw=4 et:
   2 var VM  = require('/usr/vm/node_modules/VM');
   3 var ZWatch = require('./zwatch');
   4 var common = require('./common');
   5 var crc32 = require('./crc32');
   6 var async = require('/usr/node/node_modules/async');
   7 var execFile = require('child_process').execFile;
   8 var fs = require('fs');
   9 var net = require('net');
  10 var path = require('path');
  11 var util = require('util');
  12 var zsock = require('/usr/node/node_modules/zsock');
  13 var zutil = require('/usr/node/node_modules/zutil');
  14 
  15 var sdc_fields = [
  16     'alias',
  17     'billing_id',
  18     'brand',
  19     'cpu_cap',
  20     'cpu_shares',
  21     'create_timestamp',
  22     'server_uuid',
  23     'image_uuid',
  24     'datacenter_name',
  25     'do_not_inventory',
  26     'dns_domain',
  27     'force_metadata_socket',
  28     'fs_allowed',
  29     'hostname',
  30     'limit_priv',
  31     'last_modified',
  32     'max_physical_memory',
  33     'max_locked_memory',
  34     'max_lwps',
  35     'max_swap',
  36     'nics',
  37     'owner_uuid',
  38     'package_name',
  39     'package_version',
  40     'quota',
  41     'ram',
  42     'resolvers',
  43     'routes',
  44     'state',
  45     'tmpfs',
  46     'uuid',
  47     'vcpus',
  48     'vnc_port',
  49     'zfs_io_priority',
  50     'zonepath',
  51     'zonename'
  52 ];
  53 
  54 var MetadataAgent = module.exports = function (options) {
  55     this.log = options.log;
  56     this.zlog = {};
  57     this.zones = {};
  58     this.zoneConnections = {};
  59 };
  60 
  61 MetadataAgent.prototype.createZoneLog = function (type, zonename) {
  62     var self = this;
  63     self.zlog[zonename] = self.log.child({brand: type, 'zonename': zonename});
  64     return (self.zlog[zonename]);
  65 };
  66 
  67 MetadataAgent.prototype.updateZone = function (zonename, callback) {
  68     var self = this;
  69     var log = self.log;
  70 
  71     function shouldLoad(cb) {
  72         if (!self.zones.hasOwnProperty(zonename)) {
  73             // don't have a cache, load this guy
  74             log.info('no cache for: ' + zonename + ', loading');
  75             cb(true);
  76             return;
  77         }
  78 
  79         // we do have a cached version, we'll reload only if timestamp changed.
  80         fs.stat('/etc/zones/' + zonename + '.xml', function (err, stats) {
  81             var old_mtime;
  82 
  83             if (err) {
  84                 // fail open when we can't stat
  85                 log.error({err: err}, 'cannot fs.stat() ' + zonename);
  86                 cb(true);
  87                 return;
  88             }
  89 
  90             old_mtime = (new Date(self.zones[zonename].last_modified));
  91             if (stats.mtime.getTime() > old_mtime.getTime()) {
  92                 log.info('last_modified was updated, reloading: ' + zonename);
  93                 cb(true);
  94                 return;
  95             }
  96 
  97             log.debug('using cache for: ' + zonename);
  98             cb(false);
  99         });
 100     }
 101 
 102     shouldLoad(function (load) {
 103         if (load) {
 104             VM.lookup({ zonename: zonename }, { fields: sdc_fields },
 105                 function (error, machines) {
 106                     self.zones[zonename] = machines[0];
 107                     callback();
 108                     return;
 109                 }
 110             );
 111         } else {
 112             // no need to reload since there's no change, use existing data
 113             callback();
 114             return;
 115         }
 116     });
 117 };
 118 
 119 MetadataAgent.prototype.createServersOnExistingZones = function () {
 120     var self = this;
 121 
 122     VM.lookup({}, { fields: sdc_fields }, function (error, zones) {
 123         async.forEach(zones, function (zone, cb) {
 124             if (zone.zonename === 'global') {
 125                 cb();
 126                 return;
 127             }
 128 
 129             self.zones[zone.zonename] = zone;
 130 
 131             if (zone.state !== 'running') {
 132                 self.log.debug('skipping zone ' + zone.zonename + ' which has '
 133                     + 'non-running state: ' + zone.state);
 134                 cb();
 135                 return;
 136             }
 137 
 138             if (error) {
 139                 throw error;
 140             }
 141 
 142             if (!self.zlog[zone.zonename]) {
 143                 // create a logger specific to this VM
 144                 self.createZoneLog(zone.brand, zone.zonename);
 145             }
 146 
 147             if (zone.brand === 'kvm') {
 148                 self.startKVMSocketServer(zone.zonename, function (err) {
 149                     cb();
 150                 });
 151             } else {
 152                 self.startZoneSocketServer(zone.zonename, function (err) {
 153                     cb();
 154                 });
 155             }
 156         }, function (err) {
 157             self.log.info('Created zone metadata sockets on ' + zones.length
 158                 + ' zones');
 159         });
 160     });
 161 };
 162 
 163 MetadataAgent.prototype.startDeletedZonesPurger = function () {
 164     var cmd = '/usr/sbin/zoneadm';
 165     var self = this;
 166 
 167     // Every 5 minutes we check to see whether zones we've got in self.zones
 168     // were deleted. If they are, we delete the record from the cache.
 169     setInterval(function () {
 170         execFile(cmd, ['list', '-c'], function (err, stdout, stderr) {
 171             var zones = {};
 172             if (err) {
 173                 self.log.error({err: err}, 'unable to get list of zones');
 174                 return;
 175             }
 176 
 177             // each output line is a zonename, so we turn this into an object
 178             // that looks like:
 179             //
 180             // {
 181             //   zonename: true,
 182             //   zonename: true
 183             //   ...
 184             // }
 185             //
 186             // so we can then loop through all the cached zonenames and remove
 187             // those that don't exist on the system any longer.
 188             stdout.split(/\n/).forEach(function (z) {
 189                 zones[z] = true;
 190             });
 191             Object.keys(self.zones).forEach(function (z) {
 192                 if (!zones.hasOwnProperty(z)) {
 193                     self.log.info(z + ' no longer exists, purging from cache');
 194                     delete self.zones[z];
 195                     if (self.zlog.hasOwnProperty(z)) {
 196                         delete self.zlog[z];
 197                     }
 198                 }
 199             });
 200         });
 201     }, (5 * 60 * 1000));
 202 
 203     self.log.info('Setup interval to purge deleted zones.');
 204 };
 205 
 206 MetadataAgent.prototype.start = function () {
 207     var self = this;
 208     var zwatch = this.zwatch = new ZWatch();
 209     self.createServersOnExistingZones();
 210     self.startDeletedZonesPurger();
 211 
 212     zwatch.on('zone_transition', function (msg) {
 213         if (msg.cmd === 'start') {
 214             self.updateZone(msg.zonename, function (error) {
 215                 if (error) {
 216                     self.log.error({err: error}, 'Error updating attributes: '
 217                         + error.message);
 218                     return;
 219                 }
 220                 if (!self.zlog[msg.zonename]) {
 221                     // create a logger specific to this VM
 222                     self.createZoneLog(self.zones[msg.zonename].brand,
 223                         msg.zonename);
 224                 }
 225                 if (self.zones[msg.zonename].brand === 'kvm') {
 226                     self.startKVMSocketServer(msg.zonename);
 227                 } else {
 228                     self.startZoneSocketServer(msg.zonename);
 229                 }
 230             });
 231         } else if (msg.cmd === 'stop') {
 232             if (self.zoneConnections[msg.zonename]) {
 233                 self.zoneConnections[msg.zonename].end();
 234             }
 235         }
 236     });
 237 
 238     zwatch.start(self.log);
 239 };
 240 
 241 MetadataAgent.prototype.stop = function () {
 242     this.zwatch.stop();
 243 };
 244 
 245 MetadataAgent.prototype.startKVMSocketServer = function (zonename, callback) {
 246     var self = this;
 247     var vmobj = self.zones[zonename];
 248     var zlog = self.zlog[zonename];
 249     var sockpath = path.join(vmobj.zonepath, '/root/tmp/vm.ttyb');
 250 
 251     zlog.info('Starting socket server');
 252 
 253     async.waterfall([
 254         function (cb) {
 255             common.retryUntil(2000, 120000,
 256                 function (c) {
 257                     fs.exists(sockpath, function (exists) {
 258                         zlog.debug(sockpath + ' exists: ' + exists);
 259                         setTimeout(function () {
 260                             c(null, exists);
 261                         }, 1000);
 262                     });
 263                 }, function (error) {
 264                     if (error) {
 265                         zlog.error({err: error}, 'Timed out waiting for '
 266                             + 'metadata socket');
 267                     } else {
 268                         zlog.debug('returning from startKVMSocketServer w/o '
 269                             + 'error');
 270                     }
 271                     cb(error);
 272                 }
 273             );
 274         }
 275     ], function (error) {
 276         var zopts = { zone: zonename, sockpath: sockpath };
 277         self.createKVMServer(zopts, function () {
 278             if (callback) {
 279                 callback();
 280             }
 281         });
 282     });
 283 };
 284 
 285 function rtrim(str, chars) {
 286     chars = chars || '\\s';
 287     str = str || '';
 288     return str.replace(new RegExp('[' + chars + ']+$', 'g'), '');
 289 }
 290 
 291 MetadataAgent.prototype.createKVMServer = function (zopts, callback) {
 292     var self = this;
 293     var zlog = self.zlog[zopts.zone];
 294     var kvmstream = new net.Stream();
 295 
 296     self.zoneConnections[zopts.zone] = {
 297         conn: new net.Stream(),
 298         done: false,
 299         end: function () {
 300             if (this.done) {
 301                 return;
 302             }
 303             this.done = true;
 304             zlog.info('Closing kvm stream for ' + zopts.zone);
 305             kvmstream.end();
 306         }
 307     };
 308 
 309     var buffer = '';
 310     var handler = self.makeMetadataHandler(zopts.zone, kvmstream);
 311 
 312     kvmstream.on('data', function (data) {
 313         var chunk, chunks;
 314         buffer += data.toString();
 315         chunks = buffer.split('\n');
 316         while (chunks.length > 1) {
 317             chunk = chunks.shift();
 318             handler(chunk);
 319         }
 320         buffer = chunks.pop();
 321     });
 322 
 323     kvmstream.on('error', function (e) {
 324         zlog.error({err: e}, 'KVM Socket error: ' + e.message);
 325     });
 326 
 327     kvmstream.connect(zopts.sockpath);
 328     callback();
 329 };
 330 
 331 MetadataAgent.prototype.startZoneSocketServer =
 332 function (zonename, callback) {
 333     var self = this;
 334     var zlog = self.zlog[zonename];
 335     var zonePath = self.zones[zonename].zonepath;
 336     var localpath = '/.zonecontrol';
 337     var zonecontrolpath = path.join(zonePath, 'root', localpath);
 338 
 339     zlog.info('Starting socket server');
 340 
 341     function ensureZonecontrolExists(cb) {
 342         fs.exists(zonecontrolpath, function (exists) {
 343             if (exists)  {
 344                 cb();
 345                 return;
 346             } else {
 347                 fs.mkdir(zonecontrolpath, parseInt('700', 8), function (error) {
 348                     cb(error);
 349                 });
 350             }
 351         });
 352     }
 353 
 354     ensureZonecontrolExists(function (err) {
 355         var sockpath = path.join(localpath, 'metadata.sock');
 356         var zopts = {
 357             zone: zonename,
 358             path: sockpath
 359         };
 360 
 361         if (err) {
 362             callback({err: err}, 'unable to create ' + zonecontrolpath);
 363             return;
 364         }
 365 
 366         self.createZoneSocket(zopts, function (createErr) {
 367             if (createErr) {
 368                 // We call callback here, but don't include the error because
 369                 // this is running in async.forEach and we don't want to fail
 370                 // the others and there's nothing we can do to recover anyway.
 371                 if (callback) {
 372                     callback();
 373                 }
 374                 return;
 375             }
 376 
 377             zlog.info('Zone socket created.');
 378 
 379             if (callback) {
 380                 callback();
 381             }
 382         });
 383     });
 384 };
 385 
 386 MetadataAgent.prototype.createZoneSocket =
 387 function (zopts, callback, waitSecs) {
 388     var self = this;
 389     var zlog = self.zlog[zopts.zone];
 390     waitSecs = waitSecs || 1;
 391 
 392     zsock.createZoneSocket(zopts, function (error, fd) {
 393         if (error) {
 394             // If we get errors trying to create the zone socket, wait and then
 395             // keep retrying.
 396             waitSecs = waitSecs * 2;
 397             zlog.error(
 398                 { err: error },
 399                 'createZoneSocket error, %s seconds before next attempt',
 400                 waitSecs);
 401             setTimeout(function () {
 402                 self.createZoneSocket(zopts, function () {}, waitSecs);
 403             }, waitSecs * 1000);
 404             return;
 405         }
 406 
 407         var server = net.createServer(function (socket) {
 408             var handler = self.makeMetadataHandler(zopts.zone, socket);
 409             var buffer = '';
 410             socket.on('data', function (data) {
 411                 var chunk, chunks;
 412                 buffer += data.toString();
 413                 chunks = buffer.split('\n');
 414                 while (chunks.length > 1) {
 415                     chunk = chunks.shift();
 416                     handler(chunk);
 417                 }
 418                 buffer = chunks.pop();
 419             });
 420 
 421             socket.on('error', function (err) {
 422                 zlog.error({err: err}, 'ZSocket error: ' + err.message);
 423                 zlog.info('Attempting to recover; closing and recreating zone '
 424                     + 'socket and server.');
 425                 try {
 426                     server.close();
 427                 } catch (e) {
 428                     zlog.error({err: e}, 'Caught exception closing server: '
 429                         + e.message);
 430                 }
 431 
 432                 socket.end();
 433                 self.createZoneSocket(zopts);
 434             });
 435         });
 436 
 437         self.zoneConnections[zopts.zone] = {
 438             conn: server,
 439             done: false,
 440             end: function () {
 441                 if (this.done) {
 442                     return;
 443                 }
 444                 this.done = true;
 445                 zlog.info('Closing server');
 446                 server.close();
 447             }
 448         };
 449 
 450         server.on('error', function (e) {
 451             zlog.error({err: e}, 'Zone socket error: ' + e.message);
 452             if (e.code !== 'EINTR') {
 453                 throw e;
 454             }
 455         });
 456         var Pipe = process.binding('pipe_wrap').Pipe;
 457         var p = new Pipe(true);
 458         p.open(fd);
 459         p.readable = p.writable = true;
 460         server._handle = p;
 461 
 462         server.listen();
 463     });
 464 
 465     if (callback) {
 466         callback();
 467     }
 468 };
 469 
 470 MetadataAgent.prototype.makeMetadataHandler = function (zone, socket) {
 471     var self = this;
 472     var zlog = self.zlog[zone];
 473     var write = function (str) {
 474         if (socket.writable) {
 475             socket.write(str);
 476         } else {
 477             zlog.error('Socket for ' + zone + ' closed before we could write '
 478                 + 'anything.');
 479         }
 480     };
 481 
 482     return function (data) {
 483         var cmd;
 484         var parts;
 485         var val;
 486         var vmobj;
 487         var want;
 488         var reqid;
 489         var req_is_v2 = false;
 490 
 491         parts = rtrim(data.toString()).replace(/\n$/, '')
 492             .match(/^([^\s]+)\s?(.*)/);
 493 
 494         if (!parts) {
 495             write('invalid command\n');
 496             return;
 497         }
 498 
 499         cmd = parts[1];
 500         want = parts[2];
 501 
 502         if ((cmd === 'NEGOTIATE' || cmd === 'GET') && !want) {
 503             write('invalid command\n');
 504             return;
 505         }
 506 
 507         vmobj = self.zones[zone];
 508 
 509         // Unbox V2 protocol frames:
 510         if (cmd === 'V2') {
 511             if (!parse_v2_request(want))
 512                 return;
 513         }
 514 
 515         if (cmd === 'GET') {
 516             zlog.info('Serving ' + want);
 517             if (want.slice(0, 4) === 'sdc:') {
 518                 want = want.slice(4);
 519 
 520                 // NOTE: sdc:nics, sdc:resolvers and sdc:routes are not a
 521                 // committed interface, do not rely on it. At this point it
 522                 // should only be used by mdata-fetch, if you add a consumer
 523                 // that depends on it, please add a note about that here
 524                 // otherwise expect it will be removed on you sometime.
 525                 if (want === 'nics' && vmobj.hasOwnProperty('nics')) {
 526                     val = JSON.stringify(vmobj.nics);
 527                     returnit(null, val);
 528                     return;
 529                 } else if (want === 'resolvers'
 530                     && vmobj.hasOwnProperty('resolvers')) {
 531 
 532                     // resolvers and routes are special because we might reload
 533                     // metadata trying to get the new ones w/o zone reboot. To
 534                     // ensure these are fresh we always run updateZone which
 535                     // reloads the data if stale.
 536                     self.updateZone(zone, function () {
 537                         // See NOTE above about nics, same applies to resolvers.
 538                         // It's here solely for the use of mdata-fetch.
 539                         val = JSON.stringify(self.zones[zone].resolvers);
 540                         returnit(null, val);
 541                         return;
 542                     });
 543                 } else if (want === 'routes'
 544                     && vmobj.hasOwnProperty('routes')) {
 545 
 546                     var vmRoutes = [];
 547 
 548                     self.updateZone(zone, function () {
 549 
 550                         vmobj = self.zones[zone];
 551 
 552                         // The notes above about resolvers also to routes. It's
 553                         // here solely for the use of mdata-fetch, and we need
 554                         // to do the updateZone here so that we have latest
 555                         // data.
 556                         for (var r in vmobj.routes) {
 557                             var route = { linklocal: false, dst: r };
 558                             var nicIdx = vmobj.routes[r].match(/nics\[(\d+)\]/);
 559                             if (!nicIdx) {
 560                                 // Non link-local route: we have all the
 561                                 // information we need already
 562                                 route.gateway = vmobj.routes[r];
 563                                 vmRoutes.push(route);
 564                                 continue;
 565                             }
 566                             nicIdx = Number(nicIdx[1]);
 567 
 568                             // Link-local route: we need the IP of the local nic
 569                             if (!vmobj.hasOwnProperty('nics')
 570                                 || !vmobj.nics[nicIdx]
 571                                 || !vmobj.nics[nicIdx].hasOwnProperty('ip')
 572                                 || vmobj.nics[nicIdx].ip === 'dhcp') {
 573 
 574                                 continue;
 575                             }
 576 
 577                             route.gateway = vmobj.nics[nicIdx].ip;
 578                             route.linklocal = true;
 579                             vmRoutes.push(route);
 580                         }
 581 
 582                         returnit(null, JSON.stringify(vmRoutes));
 583                         return;
 584                     });
 585                 } else if (want === 'operator-script') {
 586                     addMetadata(function (err) {
 587                         if (err) {
 588                             returnit(new Error('Unable to load metadata: '
 589                                 + err.message));
 590                             return;
 591                         }
 592 
 593                         returnit(null,
 594                             vmobj.internal_metadata['operator-script']);
 595                         return;
 596                     });
 597                 } else {
 598                     addTags(function (err) {
 599                         if (!err) {
 600                             val = VM.flatten(vmobj, want);
 601                         }
 602                         returnit(err, val);
 603                         return;
 604                     });
 605                 }
 606             } else {
 607                 // not sdc:, so key will come from *_mdata
 608                 addMetadata(function (err) {
 609                     var which_mdata = 'customer_metadata';
 610 
 611                     if (err) {
 612                         returnit(new Error('Unable to load metadata: '
 613                             + err.message));
 614                         return;
 615                     }
 616 
 617                     if (want.match(/_pw$/)) {
 618                         which_mdata = 'internal_metadata';
 619                     }
 620 
 621                     if (vmobj.hasOwnProperty(which_mdata)) {
 622                         returnit(null, vmobj[which_mdata][want]);
 623                         return;
 624                     } else {
 625                         returnit(new Error('Zone did not contain '
 626                             + which_mdata));
 627                         return;
 628                     }
 629                 });
 630             }
 631         } else if (cmd === 'NEGOTIATE') {
 632             if (want === 'V2') {
 633                 write('V2_OK\n');
 634             } else {
 635                 write('FAILURE\n');
 636             }
 637             return;
 638         } else if (req_is_v2 && cmd === 'DELETE') {
 639             want = want.trim();
 640 
 641             if (!want) {
 642                 returnit(new Error('Invalid DELETE Request'));
 643                 return;
 644             }
 645 
 646             setMetadata(want, null, function (err) {
 647                 if (err) {
 648                     returnit(err);
 649                 } else {
 650                     returnit(null, 'OK');
 651                 }
 652             });
 653         } else if (req_is_v2 && cmd === 'PUT') {
 654             var key;
 655             var value;
 656             var terms;
 657 
 658             terms = want.trim().split(' ');
 659 
 660             if (terms.length !== 2) {
 661                 returnit(new Error('Invalid PUT Request'));
 662                 return;
 663             }
 664 
 665             // PUT requests have two space-separated BASE64-encoded
 666             // arguments: the Key and then the Value.
 667             key = (new Buffer(terms[0], 'base64')).toString().trim();
 668             value = (new Buffer(terms[1], 'base64')).toString();
 669 
 670             if (!key) {
 671                 returnit(new Error('Invalid PUT Request'));
 672                 return;
 673             }
 674 
 675             if (key.slice(0, 4) === 'sdc:') {
 676                 returnit(new Error('Cannot update the "sdc" Namespace.'));
 677                 return;
 678             }
 679 
 680             zlog.info('PUT of key "' + key + '"');
 681             setMetadata(key, value, function (err) {
 682                 if (err) {
 683                     zlog.error(err, 'could not set metadata (key "' + key
 684                         + '")');
 685                     returnit(err);
 686                 } else {
 687                     returnit(null, 'OK');
 688                 }
 689             });
 690 
 691             return;
 692         } else if (cmd === 'KEYS') {
 693             addMetadata(function (err) {
 694                 var ckeys = [];
 695                 var ikeys = [];
 696 
 697                 if (err) {
 698                     returnit(new Error('Unable to load metadata: '
 699                         + err.message));
 700                     return;
 701                 }
 702 
 703                 // *_pw$ keys come from internal_metadata, everything else comes
 704                 // from customer_metadata
 705                 ckeys = Object.keys(vmobj.customer_metadata)
 706                     .filter(function (k) {
 707 
 708                     return (!k.match(/_pw$/));
 709                 });
 710                 ikeys = Object.keys(vmobj.internal_metadata)
 711                     .filter(function (k) {
 712 
 713                     return (k.match(/_pw$/));
 714                 });
 715 
 716                 returnit(null, ckeys.concat(ikeys).join('\n'));
 717                 return;
 718             });
 719         } else {
 720             zlog.error('Unknown command ' + cmd);
 721             returnit(new Error('Unknown command ' + cmd));
 722             return;
 723         }
 724 
 725         function addTags(cb) {
 726             var filename;
 727 
 728             filename = vmobj.zonepath + '/config/tags.json';
 729             fs.readFile(filename, function (err, file_data) {
 730 
 731                 if (err && err.code === 'ENOENT') {
 732                     vmobj.tags = {};
 733                     cb();
 734                     return;
 735                 }
 736 
 737                 if (err) {
 738                     zlog.error({err: err}, 'failed to load tags.json: '
 739                         + err.message);
 740                     cb(err);
 741                     return;
 742                 }
 743 
 744                 try {
 745                     vmobj.tags = JSON.parse(file_data.toString());
 746                     cb();
 747                 } catch (e) {
 748                     zlog.error({err: e}, 'unable to tags.json for ' + zone
 749                         + ': ' + e.message);
 750                     cb(e);
 751                 }
 752 
 753                 return;
 754             });
 755         }
 756 
 757         function addMetadata(cb) {
 758             var filename;
 759 
 760             // If we got here, our answer comes from metadata.
 761             // XXX In the future, if the require overhead here ends up being
 762             // larger than a stat would be, we might want to cache these and
 763             // reload when mtime changes.
 764 
 765             filename = vmobj.zonepath + '/config/metadata.json';
 766 
 767             fs.readFile(filename, function (err, file_data) {
 768                 var json = {};
 769                 var mdata_types = [ 'customer_metadata', 'internal_metadata' ];
 770 
 771                 // start w/ both empty, if we fail partway through there will
 772                 // just be no metadata instead of wrong metadata.
 773                 vmobj.customer_metadata = {};
 774                 vmobj.internal_metadata = {};
 775 
 776                 if (err && err.code === 'ENOENT') {
 777                     cb();
 778                     return;
 779                 }
 780 
 781                 if (err) {
 782                     zlog.error({err: err}, 'failed to load mdata.json: '
 783                         + err.message);
 784                     cb(err);
 785                     return;
 786                 }
 787 
 788                 try {
 789                     json = JSON.parse(file_data.toString());
 790                     mdata_types.forEach(function (mdata) {
 791                         if (json.hasOwnProperty(mdata)) {
 792                             vmobj[mdata] = json[mdata];
 793                         }
 794                     });
 795                     cb();
 796                 } catch (e) {
 797                     zlog.error({err: e}, 'unable to load metadata.json for '
 798                         + zone + ': ' + e.message);
 799                     cb(e);
 800                 }
 801 
 802                 return;
 803             });
 804         }
 805 
 806         function setMetadata(_key, _value, cb) {
 807             var payload = {};
 808             var which = 'customer_metadata';
 809 
 810             // Some keys come from "internal_metadata":
 811             if (_key.match(/_pw$/) || _key === 'operator-script') {
 812                 which = 'internal_metadata';
 813             }
 814 
 815             // Construct payload for VM.update()
 816             if (_value) {
 817                 payload['set_' + which] = {};
 818                 payload['set_' + which][_key] = _value;
 819             } else {
 820                 payload['remove_' + which] = [ _key ];
 821             }
 822 
 823             zlog.trace({ payload: payload }, 'calling VM.update()');
 824             VM.update(vmobj.uuid, payload, zlog, cb);
 825         }
 826 
 827         function parse_v2_request(inframe) {
 828             var m;
 829             var m2;
 830             var newcrc32;
 831             var framecrc32;
 832             var clen;
 833 
 834             m = inframe.match(
 835                 /\s*([0-9]+)\s+([0-9a-f]+)\s+([0-9a-f]+)\s+(.*)/);
 836             if (!m) {
 837                 zlog.error('V2 frame did not parse: ', inframe);
 838                 return (false);
 839             }
 840 
 841             clen = Number(m[1]);
 842             if (!(clen > 0) || clen !== (m[3] + ' ' + m[4]).length) {
 843                 zlog.error('V2 invalid clen: ' + m[1]);
 844                 return (false);
 845             }
 846 
 847             newcrc32 = crc32.crc32_calc(m[3] + ' ' + m[4]);
 848             framecrc32 = m[2];
 849             if (framecrc32 !== newcrc32) {
 850                 zlog.error('V2 crc mismatch (ours ' + newcrc32
 851                     + ' theirs ' + framecrc32 + '): ' + want);
 852                 return (false);
 853             }
 854 
 855             reqid = m[3];
 856 
 857             m2 = m[4].match(/\s*(\S+)\s*(.*)/);
 858             if (!m2) {
 859                 zlog.error('V2 invalid body: ' + m[4]);
 860                 return (false);
 861             }
 862 
 863             cmd = m2[1];
 864             want = (new Buffer(m2[2], 'base64')).toString('utf8');
 865 
 866             req_is_v2 = true;
 867 
 868             return (true);
 869         }
 870 
 871 
 872         function format_v2_response(code, body) {
 873             var resp;
 874             var fullresp;
 875 
 876             resp = reqid + ' ' + code;
 877             if (body)
 878                 resp += ' ' + (new Buffer(body).toString('base64'));
 879 
 880             fullresp = 'V2 ' + resp.length + ' ' + crc32.crc32_calc(
 881                 resp) + ' ' + resp + '\n';
 882 
 883             zlog.trace({ response: fullresp }, 'formatted V2 response');
 884 
 885             return (fullresp);
 886         }
 887 
 888         function returnit(error, retval) {
 889             var towrite;
 890 
 891             if (error) {
 892                 zlog.error(error.message);
 893                 if (req_is_v2)
 894                     write(format_v2_response('FAILURE', error.message));
 895                 else
 896                     write('FAILURE\n');
 897                 return;
 898             }
 899 
 900             // String value
 901             if (common.isString(retval)) {
 902                 if (req_is_v2) {
 903                     write(format_v2_response('SUCCESS', retval));
 904                 } else {
 905                     towrite = retval.replace(/^\./mg, '..');
 906                     write('SUCCESS\n' + towrite + '\n.\n');
 907                 }
 908                 return;
 909             } else if (!isNaN(retval)) {
 910                 if (req_is_v2) {
 911                     write(format_v2_response('SUCCESS', retval.toString()));
 912                 } else {
 913                     towrite = retval.toString().replace(/^\./mg, '..');
 914                     write('SUCCESS\n' + towrite + '\n.\n');
 915                 }
 916                 return;
 917             } else if (retval) {
 918                 // Non-string value
 919                 if (req_is_v2)
 920                     write(format_v2_response('FAILURE'));
 921                 else
 922                     write('FAILURE\n');
 923                 return;
 924             } else {
 925                 // Nothing to return
 926                 if (req_is_v2)
 927                     write(format_v2_response('NOTFOUND'));
 928                 else
 929                     write('NOTFOUND\n');
 930                 return;
 931             }
 932         }
 933     };
 934 };