Source code

001package org.hl7.fhir.r5.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.File;
035import java.io.FileNotFoundException;
036import java.io.FileOutputStream;
037import java.io.IOException;
038import java.io.OutputStreamWriter;
039import java.util.ArrayList;
040import java.util.HashMap;
041import java.util.List;
042import java.util.Map;
043
044import org.apache.commons.lang3.StringUtils;
045import org.hl7.fhir.exceptions.FHIRException;
046import org.hl7.fhir.r5.context.IWorkerContext.ValidationResult;
047import org.hl7.fhir.r5.formats.IParser.OutputStyle;
048import org.hl7.fhir.r5.formats.JsonParser;
049import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
050import org.hl7.fhir.r5.model.CodeableConcept;
051import org.hl7.fhir.r5.model.Coding;
052import org.hl7.fhir.r5.model.UriType;
053import org.hl7.fhir.r5.model.ValueSet;
054import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
055import org.hl7.fhir.r5.model.ValueSet.ConceptSetFilterComponent;
056import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionContainsComponent;
057import org.hl7.fhir.r5.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
058import org.hl7.fhir.r5.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
059import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
060import org.hl7.fhir.utilities.TextFile;
061import org.hl7.fhir.utilities.Utilities;
062import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
063import org.hl7.fhir.utilities.validation.ValidationOptions;
064
065import com.google.gson.JsonElement;
066import com.google.gson.JsonNull;
067import com.google.gson.JsonObject;
068import com.google.gson.JsonPrimitive;
069
070/**
071 * This implements a two level cache. 
072 *  - a temporary cache for remmbering previous local operations
073 *  - a persistent cache for rembering tx server operations
074 *  
075 * the cache is a series of pairs: a map, and a list. the map is the loaded cache, the list is the persiistent cache, carefully maintained in order for version control consistency
076 * 
077 * @author graha
078 *
079 */
080public class TerminologyCache {
081  public static final boolean TRANSIENT = false;
082  public static final boolean PERMANENT = true;
083  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
084  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
085  private static final String BREAK = "####";
086
087  public class CacheToken {
088    private String name;
089    private String key;
090    private String request;
091    public void setName(String n) {
092      if (name == null)
093        name = n;
094      else if (!n.equals(name))
095        name = NAME_FOR_NO_SYSTEM;
096    }
097  }
098
099  private class CacheEntry {
100    private String request;
101    private boolean persistent;
102    private ValidationResult v;
103    private ValueSetExpansionOutcome e;
104  }
105  
106  private class NamedCache {
107    private String name; 
108    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
109    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
110  }
111  
112
113  private Object lock;
114  private String folder;
115  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
116  private static boolean noCaching;
117  
118  // use lock from the context
119  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
120    super();
121    this.lock = lock;
122    this.folder = folder;
123    if (folder != null)
124      load();
125  }
126  
127  public void clear() {
128    caches.clear();
129  }
130  public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) {
131    CacheToken ct = new CacheToken();
132    if (code.hasSystem())
133      ct.name = getNameForSystem(code.getSystem());
134    else
135      ct.name = NAME_FOR_NO_SYSTEM;
136    JsonParser json = new JsonParser();
137    json.setOutputStyle(OutputStyle.PRETTY);
138    ValueSet vsc = getVSEssense(vs);
139    try {
140      ct.request = "{\"code\" : "+json.composeString(code, "code")+", \"valueSet\" :"+(vsc == null ? "null" : extracted(json, vsc))+(options == null ? "" : ", "+options.toJson())+"}";
141    } catch (IOException e) {
142      throw new Error(e);
143    }
144    ct.key = String.valueOf(hashNWS(ct.request));
145    return ct;
146  }
147
148  public String extracted(JsonParser json, ValueSet vsc) throws IOException {
149    String s = null;
150    if (vsc.getExpansion().getContains().size() > 1000 || vsc.getCompose().getIncludeFirstRep().getConcept().size() > 1000) {      
151      s = Integer.toString(vsc.hashCode()); // turn caching off - hack efficiency optimisation
152    } else {
153      s = json.composeString(vsc);
154    }
155    return s;
156  }
157
158  public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) {
159    CacheToken ct = new CacheToken();
160    for (Coding c : code.getCoding()) {
161      if (c.hasSystem())
162        ct.setName(getNameForSystem(c.getSystem()));
163    }
164    JsonParser json = new JsonParser();
165    json.setOutputStyle(OutputStyle.PRETTY);
166    ValueSet vsc = getVSEssense(vs);
167    try {
168      ct.request = "{\"code\" : "+json.composeString(code, "codeableConcept")+", \"valueSet\" :"+extracted(json, vsc)+(options == null ? "" : ", "+options.toJson())+"}";
169    } catch (IOException e) {
170      throw new Error(e);
171    }
172    ct.key = String.valueOf(hashNWS(ct.request));
173    return ct;
174  }
175  
176  public ValueSet getVSEssense(ValueSet vs) {
177    if (vs == null)
178      return null;
179    ValueSet vsc = new ValueSet();
180    vsc.setCompose(vs.getCompose());
181    if (vs.hasExpansion()) {
182      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
183      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
184    }
185    return vsc;
186  }
187
188  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
189    CacheToken ct = new CacheToken();
190    ValueSet vsc = getVSEssense(vs);
191    for (ConceptSetComponent inc : vs.getCompose().getInclude())
192      if (inc.hasSystem())
193        ct.setName(getNameForSystem(inc.getSystem()));
194    for (ConceptSetComponent inc : vs.getCompose().getExclude())
195      if (inc.hasSystem())
196        ct.setName(getNameForSystem(inc.getSystem()));
197    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
198      if (inc.hasSystem())
199        ct.setName(getNameForSystem(inc.getSystem()));
200    JsonParser json = new JsonParser();
201    json.setOutputStyle(OutputStyle.PRETTY);
202    try {
203      ct.request = "{\"hierarchical\" : "+(heirarchical ? "true" : "false")+", \"valueSet\" :"+extracted(json, vsc)+"}\r\n";
204    } catch (IOException e) {
205      throw new Error(e);
206    }
207    ct.key = String.valueOf(hashNWS(ct.request));
208    return ct;
209  }
210
211  private String getNameForSystem(String system) {
212    if (system.equals("http://snomed.info/sct"))
213      return "snomed";
214    if (system.equals("http://www.nlm.nih.gov/research/umls/rxnorm"))
215      return "rxnorm";
216    if (system.equals("http://loinc.org"))
217      return "loinc";
218    if (system.equals("http://unitsofmeasure.org"))
219      return "ucum";
220    if (system.startsWith("http://hl7.org/fhir/sid/"))
221      return system.substring(24).replace("/", "");
222    if (system.startsWith("urn:iso:std:iso:"))
223      return "iso"+system.substring(16).replace(":", "");
224    if (system.startsWith("http://terminology.hl7.org/CodeSystem/"))
225      return system.substring(38).replace("/", "");
226    if (system.startsWith("http://hl7.org/fhir/"))
227      return system.substring(20).replace("/", "");
228    if (system.equals("urn:ietf:bcp:47"))
229      return "lang";
230    if (system.equals("urn:ietf:bcp:13"))
231      return "mimetypes";
232    if (system.equals("urn:iso:std:iso:11073:10101"))
233      return "11073";
234    if (system.equals("http://dicom.nema.org/resources/ontology/DCM"))
235      return "dicom";
236    return system.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X");
237  }
238
239  public NamedCache getNamedCache(CacheToken cacheToken) {
240    NamedCache nc = caches.get(cacheToken.name);
241    if (nc == null) {
242      nc = new NamedCache();
243      nc.name = cacheToken.name;
244      caches.put(nc.name, nc);
245    }
246    return nc;
247  }
248  
249  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
250    synchronized (lock) {
251      NamedCache nc = getNamedCache(cacheToken);
252      CacheEntry e = nc.map.get(cacheToken.key);
253      if (e == null)
254        return null;
255      else
256        return e.e;
257    }
258  }
259
260  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
261    synchronized (lock) {      
262      NamedCache nc = getNamedCache(cacheToken);
263      CacheEntry e = new CacheEntry();
264      e.request = cacheToken.request;
265      e.persistent = persistent;
266      e.e = res;
267      store(cacheToken, persistent, nc, e);
268    }    
269  }
270
271  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
272    if (noCaching) {
273      return;
274    }
275    boolean n = nc.map.containsKey(cacheToken.key);
276    nc.map.put(cacheToken.key, e);
277    if (persistent) {
278      if (n) {
279        for (int i = nc.list.size()- 1; i>= 0; i--) {
280          if (nc.list.get(i).request.equals(e.request)) {
281            nc.list.remove(i);
282          }
283        }
284      }
285      nc.list.add(e);
286      save(nc);  
287    }
288  }
289
290  public ValidationResult getValidation(CacheToken cacheToken) {
291    if (cacheToken.key == null) {
292      return null;
293    }
294    synchronized (lock) {
295      NamedCache nc = getNamedCache(cacheToken);
296      CacheEntry e = nc.map.get(cacheToken.key);
297      if (e == null)
298        return null;
299      else
300        return e.v;
301    }
302  }
303
304  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
305    if (cacheToken.key != null) {
306      synchronized (lock) {      
307        NamedCache nc = getNamedCache(cacheToken);
308        CacheEntry e = new CacheEntry();
309        e.request = cacheToken.request;
310        e.persistent = persistent;
311        e.v = res;
312        store(cacheToken, persistent, nc, e);
313      }    
314    }
315  }
316
317  
318  // persistence
319  
320  public void save() {
321    
322  }
323  
324  private void save(NamedCache nc) {
325    if (folder == null)
326      return;
327    
328    try {
329      OutputStreamWriter sw = new OutputStreamWriter(new FileOutputStream(Utilities.path(folder, nc.name+".cache")), "UTF-8");
330      sw.write(ENTRY_MARKER+"\r\n");
331      JsonParser json = new JsonParser();
332      json.setOutputStyle(OutputStyle.PRETTY);
333      for (CacheEntry ce : nc.list) {
334        sw.write(ce.request.trim());
335        sw.write(BREAK+"\r\n");
336        if (ce.e != null) {
337          sw.write("e: {\r\n");
338          if (ce.e.getValueset() != null)
339            sw.write("  \"valueSet\" : "+json.composeString(ce.e.getValueset()).trim()+",\r\n");
340          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.e.getError()).trim()+"\"\r\n}\r\n");
341        } else {
342          sw.write("v: {\r\n");
343          boolean first = true;
344          if (ce.v.getDisplay() != null) {            
345            if (first) first = false; else sw.write(",\r\n");
346            sw.write("  \"display\" : \""+Utilities.escapeJson(ce.v.getDisplay()).trim()+"\"");
347          }
348          if (ce.v.getCode() != null) {
349            if (first) first = false; else sw.write(",\r\n");
350            sw.write("  \"code\" : \""+Utilities.escapeJson(ce.v.getCode()).trim()+"\"");
351          }
352          if (ce.v.getSystem() != null) {
353            if (first) first = false; else sw.write(",\r\n");
354            sw.write("  \"system\" : \""+Utilities.escapeJson(ce.v.getSystem()).trim()+"\"");
355          }
356          if (ce.v.getSeverity() != null) {
357            if (first) first = false; else sw.write(",\r\n");
358            sw.write("  \"severity\" : "+"\""+ce.v.getSeverity().toCode().trim()+"\""+"");
359          }
360          if (ce.v.getMessage() != null) {
361            if (first) first = false; else sw.write(",\r\n");
362            sw.write("  \"error\" : \""+Utilities.escapeJson(ce.v.getMessage()).trim()+"\"");
363          }
364          if (ce.v.getErrorClass() != null) {
365            if (first) first = false; else sw.write(",\r\n");
366            sw.write("  \"class\" : \""+Utilities.escapeJson(ce.v.getErrorClass().toString())+"\"");
367          }
368          if (ce.v.getDefinition() != null) {
369            if (first) first = false; else sw.write(",\r\n");
370            sw.write("  \"definition\" : \""+Utilities.escapeJson(ce.v.getDefinition()).trim()+"\"");
371          }
372          sw.write("\r\n}\r\n");
373        }
374        sw.write(ENTRY_MARKER+"\r\n");
375      }      
376      sw.close();
377    } catch (Exception e) {
378      System.out.println("error saving "+nc.name+": "+e.getMessage());
379    }
380  }
381
382  private void load() throws FHIRException {
383    for (String fn : new File(folder).list()) {
384      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
385        int c = 0;
386        try {
387          String title = fn.substring(0, fn.lastIndexOf("."));
388          NamedCache nc = new NamedCache();
389          nc.name = title;
390          caches.put(title, nc);
391          String src = TextFile.fileToString(Utilities.path(folder, fn));
392          if (src.startsWith("?"))
393            src = src.substring(1);
394          int i = src.indexOf(ENTRY_MARKER); 
395          while (i > -1) {
396            c++;
397            String s = src.substring(0, i);
398            src = src.substring(i+ENTRY_MARKER.length()+1);
399            i = src.indexOf(ENTRY_MARKER);
400            if (!Utilities.noString(s)) {
401              int j = s.indexOf(BREAK);
402              String q = s.substring(0, j);
403              String p = s.substring(j+BREAK.length()+1).trim();
404              CacheEntry ce = new CacheEntry();
405              ce.persistent = true;
406              ce.request = q;
407              boolean e = p.charAt(0) == 'e';
408              p = p.substring(3);
409              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
410              String error = loadJS(o.get("error"));
411              if (e) {
412                if (o.has("valueSet"))
413                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
414                else
415                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
416              } else {
417                String t = loadJS(o.get("severity"));
418                IssueSeverity severity = t == null ? null :  IssueSeverity.fromCode(t);
419                String display = loadJS(o.get("display"));
420                String code = loadJS(o.get("code"));
421                String system = loadJS(o.get("system"));
422                String definition = loadJS(o.get("definition"));
423                t = loadJS(o.get("class"));
424                TerminologyServiceErrorClass errorClass = t == null ? null : TerminologyServiceErrorClass.valueOf(t) ;
425                ce.v = new ValidationResult(severity, error, system, new ConceptDefinitionComponent().setDisplay(display).setDefinition(definition).setCode(code)).setErrorClass(errorClass);
426              }
427              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
428              nc.list.add(ce);
429            }
430          }        
431        } catch (Exception e) {
432          throw new FHIRException("Error loading "+fn+": "+e.getMessage()+" entry "+c, e);
433        }
434      }
435    }
436  }
437  
438  private String loadJS(JsonElement e) {
439    if (e == null)
440      return null;
441    if (!(e instanceof JsonPrimitive))
442      return null;
443    String s = e.getAsString();
444    if ("".equals(s))
445      return null;
446    return s;
447  }
448
449  private String hashNWS(String s) {
450    s = StringUtils.remove(s, ' ');
451    s = StringUtils.remove(s, '\n');
452    s = StringUtils.remove(s, '\r');
453    return String.valueOf(s.hashCode());
454  }
455
456  // management
457  
458  public TerminologyCache copy() {
459    // TODO Auto-generated method stub
460    return null;
461  }
462  
463  public String summary(ValueSet vs) {
464    if (vs == null)
465      return "null";
466    
467    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
468    for (ConceptSetComponent cc : vs.getCompose().getInclude())
469      b.append("Include "+getIncSummary(cc));
470    for (ConceptSetComponent cc : vs.getCompose().getExclude())
471      b.append("Exclude "+getIncSummary(cc));
472    return b.toString();
473  }
474
475  private String getIncSummary(ConceptSetComponent cc) {
476    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
477    for (UriType vs : cc.getValueSet())
478      b.append(vs.asStringValue());
479    String vsd = b.length() > 0 ? " where the codes are in the value sets ("+b.toString()+")" : "";
480    String system = cc.getSystem();
481    if (cc.hasConcept())
482      return Integer.toString(cc.getConcept().size())+" codes from "+system+vsd;
483    if (cc.hasFilter()) {
484      String s = "";
485      for (ConceptSetFilterComponent f : cc.getFilter()) {
486        if (!Utilities.noString(s))
487          s = s + " & ";
488        s = s + f.getProperty()+" "+(f.hasOp() ? f.getOp().toCode() : "?")+" "+f.getValue();
489      }
490      return "from "+system+" where "+s+vsd;
491    }
492    return "All codes from "+system+vsd;
493  }
494
495  public String summary(Coding code) {
496    return code.getSystem()+"#"+code.getCode()+": \""+code.getDisplay()+"\"";
497  }
498
499  public String summary(CodeableConcept code) {
500    StringBuilder b = new StringBuilder();
501    b.append("{");
502    boolean first = true;
503    for (Coding c : code.getCoding()) {
504      if (first) first = false; else b.append(",");
505      b.append(summary(c));
506    }
507    b.append("}: \"");
508    b.append(code.getText());
509    b.append("\"");
510    return b.toString();
511  }
512
513  public static boolean isNoCaching() {
514    return noCaching;
515  }
516
517  public static void setNoCaching(boolean noCaching) {
518    TerminologyCache.noCaching = noCaching;
519  }
520
521  public void removeCS(String url) {
522    synchronized (lock) {
523      String name = getNameForSystem(url);
524      if (caches.containsKey(name)) {
525        caches.remove(name);
526      }
527    }   
528  }
529
530  public String getFolder() {
531    return folder;
532  }
533 
534  
535}