001package org.hl7.fhir.utilities.npm;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.BufferedOutputStream;
035import java.io.ByteArrayInputStream;
036import java.io.ByteArrayOutputStream;
037import java.io.File;
038import java.io.FileInputStream;
039import java.io.FileNotFoundException;
040import java.io.IOException;
041import java.io.InputStream;
042import java.io.OutputStream;
043import java.nio.charset.StandardCharsets;
044import java.text.ParseException;
045import java.text.SimpleDateFormat;
046import java.util.ArrayList;
047import java.util.Collections;
048import java.util.Comparator;
049import java.util.Date;
050import java.util.HashMap;
051import java.util.List;
052import java.util.Map;
053import java.util.Map.Entry;
054import java.util.Set;
055import java.util.zip.ZipEntry;
056import java.util.zip.ZipInputStream;
057
058import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
059import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
060import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
061import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
062import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
063import org.hl7.fhir.exceptions.FHIRException;
064import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
065import org.hl7.fhir.utilities.SimpleHTTPClient;
066import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
067import org.hl7.fhir.utilities.TextFile;
068import org.hl7.fhir.utilities.Utilities;
069import org.hl7.fhir.utilities.json.model.JsonArray;
070import org.hl7.fhir.utilities.json.model.JsonElement;
071import org.hl7.fhir.utilities.json.model.JsonObject;
072import org.hl7.fhir.utilities.json.model.JsonProperty;
073import org.hl7.fhir.utilities.json.parser.JsonParser;
074import org.hl7.fhir.utilities.npm.PackageGenerator.PackageType;
075import org.jetbrains.annotations.NotNull;
076
077/**
078 * info and loader for a package 
079 * 
080 * Packages may exist on disk in the cache, or purely in memory when they are loaded on the fly
081 * 
082 * Packages are contained in subfolders (see the package spec). The FHIR resources will be in "package"
083 * 
084 * @author Grahame Grieve
085 *
086 */
087public class NpmPackage {
088
089  public interface ITransformingLoader {
090
091    byte[] load(File f);
092
093  }
094
095  public class PackageResourceInformationSorter implements Comparator<PackageResourceInformation> {
096    @Override
097    public int compare(PackageResourceInformation o1, PackageResourceInformation o2) {
098      return o1.filename.compareTo(o2.filename);
099    }
100  }
101  
102  public class PackageResourceInformation {
103    private String id;
104    private String resourceType;
105    private String url;
106    private String version;
107    private String filename;
108    private String supplements;
109    private String stype;
110    
111    public PackageResourceInformation(String root, JsonObject fi) throws IOException {
112      super();
113      id = fi.asString("id");
114      resourceType = fi.asString("resourceType");
115      url = fi.asString("url");
116      version = fi.asString("version");
117      filename = Utilities.path(root, fi.asString("filename"));
118      supplements = fi.asString("supplements");
119      stype = fi.asString("type");
120    }
121    public String getId() {
122      return id;
123    }
124    public String getResourceType() {
125      return resourceType;
126    }
127    public String getStatedType() {
128      return stype;
129    }
130    public String getUrl() {
131      return url;
132    }
133    public String getVersion() {
134      return version;
135    }
136    public String getFilename() {
137      return filename;
138    }
139    public String getSupplements() {
140      return supplements;
141    }
142    
143  }
144  public class IndexVersionSorter implements Comparator<JsonObject> {
145
146    @Override
147    public int compare(JsonObject o0, JsonObject o1) {
148      String v0 = o0.asString("version"); 
149      String v1 = o1.asString("version"); 
150      return v0.compareTo(v1);
151    }
152  }
153
154  public static boolean isValidName(String pid) {
155    return pid.matches("^[a-z][a-zA-Z0-9]*(\\.[a-z][a-zA-Z0-9\\-]*)+$");
156  }
157
158  public static boolean isValidVersion(String ver) {
159    return ver.matches("^[0-9]+\\.[0-9]+\\.[0-9]+$");
160  }
161
162  public class NpmPackageFolder {
163    private String name;
164    private Map<String, List<String>> types = new HashMap<>();
165    private Map<String, byte[]> content = new HashMap<>();
166    private JsonObject index;
167    private File folder;
168
169    public NpmPackageFolder(String name) {
170      super();
171      this.name = name;
172    }
173
174    public Map<String, List<String>> getTypes() {
175      return types;
176    }
177
178    public String getName() {
179      return name;
180    }
181
182    public boolean readIndex(JsonObject index) {
183      if (!index.has("index-version") || (index.asInteger("index-version") != NpmPackageIndexBuilder.CURRENT_INDEX_VERSION)) {
184        return false;
185      }
186      this.index = index;
187      for (JsonObject file : index.getJsonObjects("files")) {
188        String type = file.asString("resourceType");
189        String name = file.asString("filename");
190        if (!types.containsKey(type))
191          types.put(type, new ArrayList<>());
192        types.get(type).add(name);
193      }
194      return true;
195    }
196
197    public List<String> listFiles() {
198      List<String> res = new ArrayList<>();
199      if (folder != null) {
200        for (File f : folder.listFiles()) {
201          if (!f.isDirectory() && !Utilities.existsInList(f.getName(), "package.json", ".index.json")) {
202            res.add(f.getName());
203          }
204        }
205      } else {
206        for (String s : content.keySet()) {
207          if (!Utilities.existsInList(s, "package.json", ".index.json")) {
208            res.add(s);
209          }
210        }
211      }
212      Collections.sort(res);
213      return res;
214    }
215
216    public Map<String, byte[]> getContent() {
217      return content;
218    }
219
220    public byte[] fetchFile(String file) throws FileNotFoundException, IOException {
221      if (folder != null) {
222        File f = new File(Utilities.path(folder.getAbsolutePath(), file));
223        if (f.exists()) {
224          return TextFile.fileToBytes(f);
225        } else {
226          return null;
227        }
228      } else {
229        return content.get(file);
230      }
231    }
232
233    public boolean hasFile(String file) throws IOException {
234      if (folder != null) {
235        return new File(Utilities.path(folder.getAbsolutePath(), file)).exists();
236      } else {
237        return content.containsKey(file);
238      }
239
240    }
241
242    public String dump() {
243      return name + " ("+ (folder == null ? "null" : folder.toString())+") | "+Boolean.toString(index != null)+" | "+content.size()+" | "+types.size();
244    }
245
246    public void removeFile(String n) throws IOException {
247      if (folder != null) {
248        new File(Utilities.path(folder.getAbsolutePath(), n)).delete();
249      } else {
250        content.remove(n);
251      }
252      changedByLoader = true;      
253    }
254
255  }
256
257  private String path;
258  private JsonObject npm;
259  private Map<String, NpmPackageFolder> folders = new HashMap<>();
260  private boolean changedByLoader; // internal qa only!
261  private Map<String, Object> userData = new HashMap<>();
262
263  /**
264   * Constructor
265   */
266  private NpmPackage() {
267    super();
268  }
269
270  /**
271   * Factory method that parses a package from an extracted folder
272   */
273  public static NpmPackage fromFolder(String path) throws IOException {
274    NpmPackage res = new NpmPackage();
275    res.loadFiles(path, new File(path));
276    res.checkIndexed(path);
277    return res;
278  }
279
280  /**
281   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
282   */
283  public static NpmPackage empty(PackageGenerator thePackageGenerator) {
284    NpmPackage retVal = new NpmPackage();
285    retVal.npm = thePackageGenerator.getRootJsonObject();
286    return retVal;
287  }
288
289  /**
290   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
291   */
292  public static NpmPackage empty() {
293    NpmPackage retVal = new NpmPackage();
294    return retVal;
295  }
296
297  public Map<String, Object> getUserData() {
298    return userData;
299  }
300
301  public void loadFiles(String path, File source, String... exemptions) throws FileNotFoundException, IOException {
302    this.npm = JsonParser.parseObject(TextFile.fileToString(Utilities.path(path, "package", "package.json")));
303    this.path = path;
304    
305    File dir = new File(path);
306    for (File f : dir.listFiles()) {
307      if (!isInternalExemptFile(f) && !Utilities.existsInList(f.getName(), exemptions)) {
308        if (f.isDirectory()) {
309          String d = f.getName();
310          if (!d.equals("package")) {
311            d = Utilities.path("package", d);
312          }
313          NpmPackageFolder folder = this.new NpmPackageFolder(d);
314          folder.folder = f;
315          this.folders.put(d, folder);
316          File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
317          if (ij.exists()) {
318            try {
319              if (!folder.readIndex(JsonParser.parseObject(ij))) {
320                indexFolder(folder.getName(), folder);
321              }
322            } catch (Exception e) {
323              throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
324            }
325          }
326          loadSubFolders(dir.getAbsolutePath(), f);
327        } else {
328          NpmPackageFolder folder = this.new NpmPackageFolder(Utilities.path("package", "$root"));
329          folder.folder = dir;
330          this.folders.put(Utilities.path("package", "$root"), folder);        
331        }
332      }
333    }
334  }
335
336  public static boolean isInternalExemptFile(File f) {
337    return Utilities.existsInList(f.getName(), ".git", ".svn", ".DS_Store") || Utilities.existsInList(f.getName(), "package-list.json") ||
338        Utilities.endsWithInList(f.getName(), ".tgz");
339  }
340
341  private void loadSubFolders(String rootPath, File dir) throws IOException {
342    for (File f : dir.listFiles()) {
343      if (f.isDirectory()) {
344        String d = f.getAbsolutePath().substring(rootPath.length()+1);
345        if (!d.startsWith("package")) {
346          d = Utilities.path("package", d);
347        }
348        NpmPackageFolder folder = this.new NpmPackageFolder(d);
349        folder.folder = f;
350        this.folders.put(d, folder);
351        File ij = new File(Utilities.path(f.getAbsolutePath(), ".index.json"));
352        if (ij.exists()) {
353          try {
354            if (!folder.readIndex(JsonParser.parseObject(ij))) {
355              indexFolder(folder.getName(), folder);
356            }
357          } catch (Exception e) {
358            throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
359          }
360        }
361        loadSubFolders(rootPath, f);        
362      }
363    }    
364  }
365
366  public static NpmPackage fromFolder(String folder, PackageType defType, String... exemptions) throws IOException {
367    NpmPackage res = new NpmPackage();
368    res.loadFiles(folder, new File(folder), exemptions);
369    if (!res.folders.containsKey("package")) {
370      res.folders.put("package", res.new NpmPackageFolder("package"));
371    }
372    if (!res.folders.get("package").hasFile("package.json") && defType != null) {
373      TextFile.stringToFile("{ \"type\" : \""+defType.getCode()+"\"}", Utilities.path(res.folders.get("package").folder.getAbsolutePath(), "package.json"));
374    }
375    res.npm = JsonParser.parseObject(new String(res.folders.get("package").fetchFile("package.json")));
376    return res;
377  }
378
379  private static final int BUFFER_SIZE = 1024;
380
381  public static @NotNull NpmPackage fromPackage(InputStream tgz) throws IOException {
382    return fromPackage(tgz, null, false);
383  }
384
385  public static NpmPackage fromPackage(InputStream tgz, String desc) throws IOException {
386    return fromPackage(tgz, desc, false);
387  }
388
389  public static NpmPackage fromPackage(InputStream tgz, String desc, boolean progress) throws IOException {
390    NpmPackage res = new NpmPackage();
391    res.readStream(tgz, desc, progress);
392    return res;
393  }
394
395  public void readStream(InputStream tgz, String desc, boolean progress) throws IOException {
396    GzipCompressorInputStream gzipIn;
397    try {
398      gzipIn = new GzipCompressorInputStream(tgz);
399    } catch (Exception e) {
400      throw new IOException("Error reading "+(desc == null ? "package" : desc)+": "+e.getMessage(), e);      
401    }
402    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
403      TarArchiveEntry entry;
404
405      int i = 0;
406      int c = 12;
407      while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
408        i++;
409        String n = entry.getName();
410        if (n.contains("..")) {
411          throw new RuntimeException("Entry with an illegal name: " + n);
412        }
413        if (entry.isDirectory()) {
414          String dir = n.substring(0, n.length()-1);
415          if (dir.startsWith("package/")) {
416            dir = dir.substring(8);
417          }
418          folders.put(dir, new NpmPackageFolder(dir));
419        } else {
420          int count;
421          byte data[] = new byte[BUFFER_SIZE];
422          ByteArrayOutputStream fos = new ByteArrayOutputStream();
423          try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
424            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
425              dest.write(data, 0, count);
426            }
427          }
428          fos.close();
429          loadFile(n, fos.toByteArray());
430        }
431        if (progress && i % 50 == 0) {
432          c++;
433          System.out.print(".");
434          if (c == 120) {
435            System.out.println("");
436            System.out.print("  ");
437            c = 2;
438          }
439        }
440      }
441    } 
442    try {
443      npm = JsonParser.parseObject(folders.get("package").fetchFile("package.json"));
444    } catch (Exception e) {
445      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
446    }
447    checkIndexed(desc);
448  }
449
450  public void loadFile(String n, byte[] data) throws IOException {
451    String dir = n.contains("/") ? n.substring(0, n.lastIndexOf("/")) : "$root";
452    if (dir.startsWith("package/")) {
453      dir = dir.substring(8);
454    }
455    n = n.substring(n.lastIndexOf("/")+1);
456    NpmPackageFolder index = folders.get(dir);
457    if (index == null) {
458      index = new NpmPackageFolder(dir);
459      folders.put(dir, index);
460    }
461    index.content.put(n, data);
462  }
463
464  private void checkIndexed(String desc) throws IOException {
465    for (NpmPackageFolder folder : folders.values()) {
466      if (folder.index == null) {
467        indexFolder(desc, folder);
468      }
469    }
470  }
471
472  public void indexFolder(String desc, NpmPackageFolder folder) throws FileNotFoundException, IOException {
473    List<String> remove = new ArrayList<>();
474    NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
475    indexer.start();
476    for (String n : folder.listFiles()) {
477      if (!indexer.seeFile(n, folder.fetchFile(n))) {
478        remove.add(n);
479      }
480    } 
481    for (String n : remove) {
482      folder.removeFile(n);
483    }
484    String json = indexer.build();
485    try {
486      folder.readIndex(JsonParser.parseObject(json));
487      if (folder.folder != null) {
488        TextFile.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json"));
489      }
490    } catch (Exception e) {
491      TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
492      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.name+"/.index.json: "+e.getMessage(), e);
493    }
494  }
495
496
497  public static NpmPackage fromZip(InputStream stream, boolean dropRootFolder, String desc) throws IOException {
498    NpmPackage res = new NpmPackage();
499    ZipInputStream zip = new ZipInputStream(stream);
500    ZipEntry ze;
501    while ((ze = zip.getNextEntry()) != null) {
502      int size;
503      byte[] buffer = new byte[2048];
504
505      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
506      BufferedOutputStream bos = new BufferedOutputStream(bytes, buffer.length);
507
508      while ((size = zip.read(buffer, 0, buffer.length)) != -1) {
509        bos.write(buffer, 0, size);
510      }
511      bos.flush();
512      bos.close();
513      if (bytes.size() > 0) {
514        if (dropRootFolder) {
515          res.loadFile(ze.getName().substring(ze.getName().indexOf("/")+1), bytes.toByteArray());
516        } else {
517          res.loadFile(ze.getName(), bytes.toByteArray());
518        }
519      }
520      zip.closeEntry();
521    }
522    zip.close();         
523    try {
524      res.npm = JsonParser.parseObject(res.folders.get("package").fetchFile("package.json"));
525    } catch (Exception e) {
526      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
527    }
528    res.checkIndexed(desc);
529    return res;
530  }
531
532
533  /**
534   * Accessing the contents of the package - get a list of files in a subfolder of the package 
535   *
536   * @param folder
537   * @return
538   * @throws IOException 
539   */
540  public List<String> list(String folder) throws IOException {
541    List<String> res = new ArrayList<String>();
542    if (folders.containsKey(folder)) {
543      res.addAll(folders.get(folder).listFiles());
544    } else if (folders.containsKey(Utilities.path("package", folder))) {
545      res.addAll(folders.get(Utilities.path("package", folder)).listFiles());
546    }
547    return res;
548  }
549
550  public List<String> listResources(String... types) throws IOException {
551    List<String> res = new ArrayList<String>();
552    NpmPackageFolder folder = folders.get("package");
553    for (String s : types) {
554      if (folder.types.containsKey(s))
555        res.addAll(folder.types.get(s));
556    }
557    Collections.sort(res);
558    return res;
559  }
560
561  public List<PackageResourceInformation> listIndexedResources(String... types) throws IOException {
562    List<PackageResourceInformation> res = new ArrayList<PackageResourceInformation>();
563    for (NpmPackageFolder folder : folders.values()) {
564      if (folder.index != null) {
565        for (JsonObject fi : folder.index.getJsonObjects("files")) {
566          if (Utilities.existsInList(fi.asString("resourceType"), types)) {
567            res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getName() : folder.folder.getAbsolutePath(), fi));
568          }
569        }
570      }
571    } 
572    //    Collections.sort(res, new PackageResourceInformationSorter());
573    return res;
574  }
575
576  /**
577   * use the name from listResources()
578   * 
579   * @param id
580   * @return
581   * @throws IOException
582   */
583  public InputStream loadResource(String file) throws IOException {
584    NpmPackageFolder folder = folders.get("package");
585    return new ByteArrayInputStream(folder.fetchFile(file));
586  }
587
588  /**
589   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
590   * 
591   * @param url - the canonical URL of the resource (exact match only)
592   * @return null if it is not found
593   * @throws IOException
594   */
595  public InputStream loadByCanonical(String canonical) throws IOException {
596    return loadByCanonicalVersion("package", canonical, null);    
597  }
598  
599  /**
600   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
601   * 
602   * @param folder - one of the folders in the package (main folder is "package")
603   * @param url - the canonical URL of the resource (exact match only)
604   * @return null if it is not found
605   * @throws IOException
606   */
607  public InputStream loadByCanonical(String folder, String canonical) throws IOException {
608    return loadByCanonicalVersion(folder, canonical, null);    
609  }
610    
611  /**
612   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
613   * 
614   * @param url - the canonical URL of the resource (exact match only)
615   * @param version - the specified version (or null if the most recent)
616   * 
617   * @return null if it is not found
618   * @throws IOException
619   */
620  public InputStream loadByCanonicalVersion(String canonical, String version) throws IOException {
621    return loadByCanonicalVersion("package", canonical, version);
622  }
623  
624  /**
625   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
626   * 
627   * @param folder - one of the folders in the package (main folder is "package")
628   * @param url - the canonical URL of the resource (exact match only)
629   * @param version - the specified version (or null if the most recent)
630   * 
631   * @return null if it is not found
632   * @throws IOException
633   */
634  public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException {
635    NpmPackageFolder f = folders.get(folder);
636    List<JsonObject> matches = new ArrayList<>();
637    for (JsonObject file : f.index.getJsonObjects("files")) {
638      if (canonical.equals(file.asString("url"))) {
639        if (version != null && version.equals(file.asString("version"))) {
640          return load("package", file.asString("filename"));
641        } else if (version == null) {
642          matches.add(file);
643        }
644      }
645      if (matches.size() > 0) {
646        if (matches.size() == 1) {
647          return load("package", matches.get(0).asString("filename"));          
648        } else {
649          Collections.sort(matches, new IndexVersionSorter());
650          return load("package", matches.get(matches.size()-1).asString("filename"));          
651        }
652      }
653    }
654    return null;        
655  }
656    
657  /**
658   * get a stream that contains the contents of one of the files in the base package
659   * 
660   * @param file
661   * @return
662   * @throws IOException
663   */
664  public InputStream load(String file) throws IOException {
665    return load("package", file);
666  }
667  /**
668   * get a stream that contains the contents of one of the files in a folder
669   * 
670   * @param folder
671   * @param file
672   * @return
673   * @throws IOException
674   */
675  public InputStream load(String folder, String file) throws IOException {
676    NpmPackageFolder f = folders.get(folder);
677    if (f == null) {
678      f = folders.get(Utilities.path("package", folder));
679    }
680    if (f != null && f.hasFile(file)) {
681      return new ByteArrayInputStream(f.fetchFile(file));
682    } else {
683      throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name());
684    }
685  }
686
687  public boolean hasFile(String folder, String file) throws IOException {
688    NpmPackageFolder f = folders.get(folder);
689    if (f == null) {
690      f = folders.get(Utilities.path("package", folder));
691    }
692    return f != null && f.hasFile(file);
693  }
694
695
696  /**
697   * Handle to the package json file
698   * 
699   * @return
700   */
701  public JsonObject getNpm() {
702    return npm;
703  }
704
705  /**
706   * convenience method for getting the package name
707   * @return
708   */
709  public String name() {
710    return npm.asString("name");
711  }
712
713  /**
714   * convenience method for getting the package id (which in NPM language is the same as the name)
715   * @return
716   */
717  public String id() {
718    return npm.asString("name");
719  }
720
721  public String date() {
722    return npm.asString("date");
723  }
724
725  public String canonical() {
726    return npm.asString("canonical");
727  }
728
729  /**
730   * convenience method for getting the package version
731   * @return
732   */
733  public String version() {
734    return npm.asString("version");
735  }
736
737  /**
738   * convenience method for getting the package fhir version
739   * @return
740   */
741  public String fhirVersion() {
742    if ("hl7.fhir.core".equals(npm.asString("name")))
743      return npm.asString("version");
744    else if (npm.asString("name").startsWith("hl7.fhir.r2.") || npm.asString("name").startsWith("hl7.fhir.r2b.") || npm.asString("name").startsWith("hl7.fhir.r3.") || 
745        npm.asString("name").startsWith("hl7.fhir.r4.") || npm.asString("name").startsWith("hl7.fhir.r4b.") || npm.asString("name").startsWith("hl7.fhir.r5."))
746      return npm.asString("version");
747    else {
748      JsonObject dep = null;
749      if (npm.hasObject("dependencies")) {
750        dep = npm.getJsonObject("dependencies");
751        if (dep != null) {
752          for (JsonProperty e : dep.getProperties()) {
753            if (Utilities.existsInList(e.getName(), "hl7.fhir.r2.core", "hl7.fhir.r2b.core", "hl7.fhir.r3.core", "hl7.fhir.r4.core"))
754              return e.getValue().asString();
755            if (Utilities.existsInList(e.getName(), "hl7.fhir.core")) // while all packages are updated
756              return e.getValue().asString();
757          }
758        }
759      }
760      if (npm.hasArray("fhirVersions")) {
761        JsonArray e = npm.getJsonArray("fhirVersions");
762        if (e.size() > 0) {
763          return e.getItems().get(0).asString();
764        }
765      }
766      if (dep != null) {
767        // legacy simplifier support:
768        if (dep.has("simplifier.core.r4"))
769          return "4.0";
770        if (dep.has("simplifier.core.r3"))
771          return "3.0";
772        if (dep.has("simplifier.core.r2"))
773          return "2.0";
774      }
775      throw new FHIRException("no core dependency or FHIR Version found in the Package definition");
776    }
777  }
778
779  public String summary() {
780    if (path != null)
781      return path;
782    else
783      return "memory";
784  }
785
786  public boolean isType(PackageType template) {
787    return template.getCode().equals(type()) || template.getOldCode().equals(type()) ;
788  }
789
790  public String type() {
791    return npm.asString("type");
792  }
793
794  public String description() {
795    return npm.asString("description");
796  }
797
798  public String getPath() {
799    return path;
800  }
801
802  public List<String> dependencies() {
803    List<String> res = new ArrayList<>();
804    if (npm.has("dependencies")) {
805      for (JsonProperty e : npm.getJsonObject("dependencies").getProperties()) {
806        res.add(e.getName()+"#"+e.getValue().asString());
807      }
808    }
809    return res;
810  }
811
812  public String homepage() {
813    return npm.asString("homepage");
814  }
815
816  public String url() {
817    return npm.asString("url");
818  }
819
820
821  public String title() {
822    return npm.asString("title");
823  }
824
825  public String toolsVersion() {
826    return npm.asString("tools-version");
827  }
828
829  public String license() {
830    return npm.asString("license");
831  }
832
833  //  /**
834  //   * only for use by the package manager itself
835  //   * 
836  //   * @param path
837  //   */
838  //  public void setPath(String path) {
839  //    this.path = path;
840  //  }
841
842  public String getWebLocation() {
843    if (npm.hasPrimitive("url")) {
844      return PackageHacker.fixPackageUrl(npm.asString("url"));
845    } else {
846      return npm.asString("canonical");
847    }
848  }
849
850  public InputStream loadResource(String type, String id) throws IOException {
851    NpmPackageFolder f = folders.get("package");
852    JsonArray files = f.index.getJsonArray("files");
853    for (JsonElement e : files.getItems()) {
854      JsonObject i = (JsonObject) e;
855      if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
856        return load("package", i.asString("filename"));
857      }
858    }
859    return null;
860  }
861
862  public InputStream loadExampleResource(String type, String id) throws IOException {
863    NpmPackageFolder f = folders.get("example");
864    if (f == null) {
865      f = folders.get("package/example");      
866    }
867    if (f != null) {
868      JsonArray files = f.index.getJsonArray("files");
869      for (JsonElement e : files.getItems()) {
870        JsonObject i = (JsonObject) e;
871        if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
872          return load("example", i.asString("filename"));
873        }
874      }
875    }
876    return null;
877  }
878
879  /** special case when playing around inside the package **/
880  public Map<String, NpmPackageFolder> getFolders() {
881    return folders;
882  }
883
884  public void save(File directory) throws IOException {
885    File dir = new File(Utilities.path(directory.getAbsolutePath(), name()));
886    if (!dir.exists()) {
887      Utilities.createDirectory(dir.getAbsolutePath());
888    } else {
889      Utilities.clearDirectory(dir.getAbsolutePath());
890    }
891    
892    for (NpmPackageFolder folder : folders.values()) {
893      String n = folder.name;
894
895      File pd = new File(Utilities.path(dir.getAbsolutePath(), n));
896      if (!pd.exists()) {
897        Utilities.createDirectory(pd.getAbsolutePath());
898      }
899      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
900      indexer.start();
901      for (String s : folder.content.keySet()) {
902        byte[] b = folder.content.get(s);
903        indexer.seeFile(s, b);
904        if (!s.equals(".index.json") && !s.equals("package.json")) {
905          TextFile.bytesToFile(b, Utilities.path(dir.getAbsolutePath(), n, s));
906        }
907      }
908      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
909      TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), n, ".index.json"));
910    }
911    byte[] cnt = TextFile.stringToBytes(JsonParser.compose(npm, true), false);
912    TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), "package", "package.json"));
913  }
914  
915  public void save(OutputStream stream) throws IOException {
916    TarArchiveOutputStream tar;
917    ByteArrayOutputStream OutputStream;
918    BufferedOutputStream bufferedOutputStream;
919    GzipCompressorOutputStream gzipOutputStream;
920
921    OutputStream = new ByteArrayOutputStream();
922    bufferedOutputStream = new BufferedOutputStream(OutputStream);
923    gzipOutputStream = new GzipCompressorOutputStream(bufferedOutputStream);
924    tar = new TarArchiveOutputStream(gzipOutputStream);
925
926
927    for (NpmPackageFolder folder : folders.values()) {
928      String n = folder.name;
929      if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) {
930        n = "package/"+n;
931      }
932      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
933      indexer.start();
934      for (String s : folder.content.keySet()) {
935        byte[] b = folder.content.get(s);
936        String name = n+"/"+s;
937        if (b == null) {
938          System.out.println(name+" is null");
939        } else {
940          indexer.seeFile(s, b);
941          if (!s.equals(".index.json") && !s.equals("package.json")) {
942            TarArchiveEntry entry = new TarArchiveEntry(name);
943            entry.setSize(b.length);
944            tar.putArchiveEntry(entry);
945            tar.write(b);
946            tar.closeArchiveEntry();
947          }
948        }
949      }
950      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
951      TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json");
952      entry.setSize(cnt.length);
953      tar.putArchiveEntry(entry);
954      tar.write(cnt);
955      tar.closeArchiveEntry();
956    }
957    byte[] cnt = TextFile.stringToBytes(JsonParser.compose(npm, true), false);
958    TarArchiveEntry entry = new TarArchiveEntry("package/package.json");
959    entry.setSize(cnt.length);
960    tar.putArchiveEntry(entry);
961    tar.write(cnt);
962    tar.closeArchiveEntry();
963
964    tar.finish();
965    tar.close();
966    gzipOutputStream.close();
967    bufferedOutputStream.close();
968    OutputStream.close();
969    byte[] b = OutputStream.toByteArray();
970    stream.write(b);
971  }
972
973  /**
974   * Keys are resource type names, values are filenames
975   */
976  public Map<String, List<String>> getTypes() {
977    return folders.get("package").types;
978  }
979
980  public String fhirVersionList() {
981    if (npm.has("fhirVersions")) {
982      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
983      if (npm.hasArray("fhirVersions")) {
984        for (String n : npm.getJsonArray("fhirVersions").asStrings()) {
985          b.append(n);
986        }
987      }
988      if (npm.hasPrimitive("fhirVersions")) {
989        b.append(npm.asString("fhirVersions"));
990      }
991      return b.toString();
992    } else
993      return "";
994  }
995
996  public String dependencySummary() {
997    if (npm.has("dependencies")) {
998      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
999      for (JsonProperty e : npm.getJsonObject("dependencies").getProperties()) {
1000        b.append(e.getName()+"#"+e.getValue().asString());
1001      }
1002      return b.toString();
1003    } else
1004      return "";
1005  }
1006
1007  public void unPack(String dir) throws IOException {
1008    unPack (dir, false);
1009  }
1010
1011  public void unPackWithAppend(String dir) throws IOException {
1012    unPack (dir, true);
1013  }
1014
1015  public void unPack(String dir, boolean withAppend) throws IOException {
1016    for (NpmPackageFolder folder : folders.values()) {
1017      String dn = folder.getName();
1018      if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) {
1019        dn = dn.substring(8);
1020      }
1021      if (dn.equals("$root")) {
1022        dn = dir;
1023      } else {
1024         dn = Utilities.path(dir, dn);
1025      }
1026      Utilities.createDirectory(dn);
1027      for (String s : folder.listFiles()) {
1028        String fn = Utilities.path(dn, s);
1029        File f = new File(fn);
1030        if (withAppend && f.getName().startsWith("_append.")) {
1031          String appendFn = Utilities.path(dn, s.substring(8));
1032          if (new File(appendFn).exists())
1033            TextFile.appendBytesToFile(folder.fetchFile(s), appendFn);        
1034          else
1035            TextFile.bytesToFile(folder.fetchFile(s), appendFn);        
1036        } else
1037          TextFile.bytesToFile(folder.fetchFile(s), fn);
1038      }
1039//      if (path != null)
1040//        FileUtils.copyDirectory(new File(path), new File(dir));      
1041    }
1042  }
1043
1044  public void debugDump(String purpose) {
1045//    System.out.println("Debug Dump of Package for '"+purpose+"'. Path = "+path);
1046//    System.out.println("  npm = "+name()+"#"+version()+", canonical = "+canonical());
1047//    System.out.println("  folders = "+folders.size());
1048//    for (String s : sorted(folders.keySet())) {
1049//      NpmPackageFolder folder = folders.get(s);
1050//      System.out.println("    "+folder.dump());
1051//    }
1052  }
1053
1054  private List<String> sorted(Set<String> keys) {
1055    List<String> res = new ArrayList<String>();
1056    res.addAll(keys);
1057    Collections.sort(res);
1058    return res ;
1059  }
1060
1061  public void clearFolder(String folderName) {
1062    NpmPackageFolder folder = folders.get(folderName);
1063    folder.content.clear();
1064    folder.types.clear();    
1065  }
1066
1067  public void deleteFolder(String folderName) {
1068    folders.remove(folderName);
1069  }
1070
1071  public void addFile(String folderName, String name, byte[] cnt, String type) {
1072    if (!folders.containsKey(folderName)) {
1073      folders.put(folderName, new NpmPackageFolder(folderName));
1074    }
1075    NpmPackageFolder folder = folders.get(folderName);
1076    folder.content.put(name, cnt);
1077    if (!folder.types.containsKey(type))
1078      folder.types.put(type, new ArrayList<>());
1079    folder.types.get(type).add(name);
1080    if ("package".equals(folderName) && "package.json".equals(name)) {
1081      try {
1082        npm = JsonParser.parseObject(cnt);
1083      } catch (IOException e) {
1084      }
1085    }
1086  }
1087
1088  public void loadAllFiles() throws IOException {
1089    for (String folder : folders.keySet()) {
1090      NpmPackageFolder pf = folders.get(folder);
1091      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1092      File file = new File(p);
1093      if (file.exists()) {
1094        for (File f : file.listFiles()) {
1095          if (!f.isDirectory() && !isInternalExemptFile(f)) {
1096            pf.getContent().put(f.getName(), TextFile.fileToBytes(f));
1097          }
1098        }
1099      }
1100    }
1101  }
1102
1103  public void loadAllFiles(ITransformingLoader loader) throws IOException {
1104    for (String folder : folders.keySet()) {
1105      NpmPackageFolder pf = folders.get(folder);
1106      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1107      for (File f : new File(p).listFiles()) {
1108        if (!f.isDirectory() && !isInternalExemptFile(f)) {
1109          pf.getContent().put(f.getName(), loader.load(f));
1110        }
1111      }
1112    }
1113  }
1114
1115  public boolean isChangedByLoader() {
1116    return changedByLoader;
1117  }
1118
1119  public boolean isCore() {
1120    return Utilities.existsInList(npm.asString("type"), "fhir.core", "Core");
1121  }
1122
1123  public boolean isTx() {
1124    return npm.asString("name").startsWith("hl7.terminology");
1125  }
1126
1127  public boolean hasCanonical(String url) {
1128    if (url == null) {
1129      return false;
1130    }
1131    String u = url.contains("|") ?  url.substring(0, url.indexOf("|")) : url;
1132    String v = url.contains("|") ?  url.substring(url.indexOf("|")+1) : null;
1133    NpmPackageFolder folder = folders.get("package");
1134    if (folder != null) {
1135      for (JsonObject o : folder.index.getJsonObjects("files")) {
1136        if (u.equals(o.asString("url"))) {
1137          if (v == null || v.equals(o.asString("version"))) {
1138            return true;
1139          }
1140        }
1141      }
1142    }
1143    return false;
1144  }
1145
1146  public boolean canLazyLoad() throws IOException {
1147    for (NpmPackageFolder folder : folders.values()) {
1148      if (folder.folder == null) {        
1149        return false;
1150      }
1151    }
1152    if (Utilities.existsInList(name(), "fhir.test.data.r2", "fhir.test.data.r3", "fhir.test.data.r4", "fhir.tx.support.r2", "fhir.tx.support.r3", "fhir.tx.support.r4", "us.nlm.vsac")) {
1153      return true;
1154    }
1155    if (npm.asBoolean("lazy-load")) {
1156      return true;
1157    }
1158    if (!hasFile("other", "spec.internals")) {
1159      return false;
1160    }
1161    return true;
1162  }
1163
1164  public boolean isNotForPublication() {
1165    return npm.asBoolean("notForPublication");
1166 }
1167
1168  public InputStream load(PackageResourceInformation p) throws FileNotFoundException {
1169    if (p.filename.startsWith("@")) {
1170      String[] pl = p.filename.substring(1).split("\\/");
1171      return new ByteArrayInputStream(folders.get(pl[0]).content.get(pl[1]));
1172    } else {
1173      return new FileInputStream(p.filename);
1174    }
1175  }
1176
1177  public Date dateAsDate() {
1178    try {
1179      String d = date();
1180      if (d == null) {
1181        switch (name()) {
1182        case "hl7.fhir.r2.core":  d = "20151024000000"; break;
1183        case "hl7.fhir.r2b.core": d = "20160330000000"; break;
1184        case "hl7.fhir.r3.core":  d = "20191024000000"; break;
1185        case "hl7.fhir.r4.core":  d = "20191030000000"; break;
1186        case "hl7.fhir.r4b.core": d = "202112200000000"; break;
1187        case "hl7.fhir.r5.core":  d = "20211219000000"; break;
1188        default:
1189          return new Date();
1190        }
1191      }
1192      return new SimpleDateFormat("yyyyMMddHHmmss").parse(d);
1193    } catch (ParseException e) {
1194      // this really really shouldn't happen
1195      return new Date();
1196    }
1197  }
1198
1199  public static NpmPackage fromUrl(String source) throws IOException {
1200    SimpleHTTPClient fetcher = new SimpleHTTPClient();
1201    HTTPResult res = fetcher.get(source+"?nocache=" + System.currentTimeMillis());
1202    res.checkThrowException();
1203    return fromPackage(new ByteArrayInputStream(res.getContent()));
1204  }
1205
1206  @Override
1207  public String toString() {
1208    return "NpmPackage "+name()+"#"+version()+" [path=" + path + "]";
1209  }
1210  
1211  
1212}