001package org.hl7.fhir.dstu3.terminologies;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import org.apache.commons.lang3.NotImplementedException;
035import org.hl7.fhir.dstu3.context.IWorkerContext;
036import org.hl7.fhir.dstu3.model.*;
037import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode;
038import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
039import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionDesignationComponent;
040import org.hl7.fhir.dstu3.model.ValueSet.*;
041import org.hl7.fhir.dstu3.utils.ToolingExtensions;
042import org.hl7.fhir.exceptions.FHIRException;
043import org.hl7.fhir.exceptions.FHIRFormatError;
044import org.hl7.fhir.exceptions.NoTerminologyServiceException;
045import org.hl7.fhir.exceptions.TerminologyServiceException;
046import org.hl7.fhir.utilities.Utilities;
047
048import java.io.FileNotFoundException;
049import java.io.IOException;
050import java.util.*;
051
052import static org.apache.commons.lang3.StringUtils.isNotBlank;
053
054/*
055 * Copyright (c) 2011+, HL7, Inc
056 * All rights reserved.
057 *
058 * Redistribution and use in source and binary forms, with or without modification,
059 * are permitted provided that the following conditions are met:
060 *
061 * Redistributions of source code must retain the above copyright notice, this
062 * list of conditions and the following disclaimer.
063 * Redistributions in binary form must reproduce the above copyright notice,
064 * this list of conditions and the following disclaimer in the documentation
065 * and/or other materials provided with the distribution.
066 * Neither the name of HL7 nor the names of its contributors may be used to
067 * endorse or promote products derived from this software without specific
068 * prior written permission.
069 *
070 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
071 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
072 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
073 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
074 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
075 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
076 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
077 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
078 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
079 * POSSIBILITY OF SUCH DAMAGE.
080 *
081 */
082
083public class ValueSetExpanderSimple implements ValueSetExpander {
084
085  private List<ValueSetExpansionContainsComponent> codes = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
086  private List<ValueSetExpansionContainsComponent> roots = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
087  private Map<String, ValueSetExpansionContainsComponent> map = new HashMap<String, ValueSet.ValueSetExpansionContainsComponent>();
088  private IWorkerContext context;
089  private boolean canBeHeirarchy = true;
090  private Set<String> excludeKeys = new HashSet<String>();
091  private Set<String> excludeSystems = new HashSet<String>();
092  private ValueSetExpanderFactory factory;
093  private ValueSet focus;
094  private int maxExpansionSize = 500;
095
096  private int total;
097
098  public ValueSetExpanderSimple(IWorkerContext context, ValueSetExpanderFactory factory) {
099    super();
100    this.context = context;
101    this.factory = factory;
102  }
103
104  public void setMaxExpansionSize(int theMaxExpansionSize) {
105    maxExpansionSize = theMaxExpansionSize;
106  }
107
108  private ValueSetExpansionContainsComponent addCode(String system, String code, String display, ValueSetExpansionContainsComponent parent, List<ConceptDefinitionDesignationComponent> designations,
109                                                     ExpansionProfile profile, boolean isAbstract, boolean inactive, List<ValueSet> filters) {
110    if (filters != null && !filters.isEmpty() && !filterContainsCode(filters, system, code))
111      return null;
112    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
113    n.setSystem(system);
114    n.setCode(code);
115    if (isAbstract)
116      n.setAbstract(true);
117    if (inactive)
118      n.setInactive(true);
119
120    if (profile.getIncludeDesignations() && designations != null) {
121      for (ConceptDefinitionDesignationComponent t : designations) {
122        ToolingExtensions.addLanguageTranslation(n, t.getLanguage(), t.getValue());
123      }
124    }
125    ConceptDefinitionDesignationComponent t = profile.hasLanguage() ? getMatchingLang(designations, profile.getLanguage()) : null;
126    if (t == null)
127      n.setDisplay(display);
128    else
129      n.setDisplay(t.getValue());
130
131    String s = key(n);
132    if (map.containsKey(s) || excludeKeys.contains(s)) {
133      canBeHeirarchy = false;
134    } else {
135      codes.add(n);
136      map.put(s, n);
137      total++;
138    }
139    if (canBeHeirarchy && parent != null) {
140      parent.getContains().add(n);
141    } else {
142      roots.add(n);
143    }
144    return n;
145  }
146
147  private boolean filterContainsCode(List<ValueSet> filters, String system, String code) {
148    for (ValueSet vse : filters)
149      if (expansionContainsCode(vse.getExpansion().getContains(), system, code))
150        return true;
151    return false;
152  }
153
154  private boolean expansionContainsCode(List<ValueSetExpansionContainsComponent> contains, String system, String code) {
155    for (ValueSetExpansionContainsComponent cc : contains) {
156      if (system.equals(cc.getSystem()) && code.equals(cc.getCode()))
157        return true;
158      if (expansionContainsCode(cc.getContains(), system, code))
159        return true;
160    }
161    return false;
162  }
163
164  private ConceptDefinitionDesignationComponent getMatchingLang(List<ConceptDefinitionDesignationComponent> list, String lang) {
165    for (ConceptDefinitionDesignationComponent t : list)
166      if (t.getLanguage().equals(lang))
167        return t;
168    for (ConceptDefinitionDesignationComponent t : list)
169      if (t.getLanguage().startsWith(lang))
170        return t;
171    return null;
172  }
173
174  private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def, ValueSetExpansionContainsComponent parent, ExpansionProfile profile, List<ValueSet> filters)
175    throws FHIRException {
176    if (!CodeSystemUtilities.isDeprecated(cs, def)) {
177      ValueSetExpansionContainsComponent np = null;
178      boolean abs = CodeSystemUtilities.isNotSelectable(cs, def);
179      boolean inc = CodeSystemUtilities.isInactive(cs, def);
180      if (canBeHeirarchy || !abs)
181        np = addCode(system, def.getCode(), def.getDisplay(), parent, def.getDesignation(), profile, abs, inc, filters);
182      for (ConceptDefinitionComponent c : def.getConcept())
183        addCodeAndDescendents(cs, system, c, np, profile, filters);
184    } else
185      for (ConceptDefinitionComponent c : def.getConcept())
186        addCodeAndDescendents(cs, system, c, null, profile, filters);
187
188  }
189
190  private void addCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile, List<ValueSet> filters) throws ETooCostly {
191    if (expand.getContains().size() > maxExpansionSize)
192      throw new ETooCostly("Too many codes to display (>" + Integer.toString(expand.getContains().size()) + ")");
193    for (ValueSetExpansionParameterComponent p : expand.getParameter()) {
194      if (!existsInParams(params, p.getName(), p.getValue()))
195        params.add(p);
196    }
197
198    copyImportContains(expand.getContains(), null, profile, filters);
199  }
200
201  private void excludeCode(String theSystem, String theCode) {
202    ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
203    n.setSystem(theSystem);
204    n.setCode(theCode);
205    String s = key(n);
206    excludeKeys.add(s);
207  }
208
209  private void excludeCodes(ConceptSetComponent exc, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException {
210    if (exc.hasSystem() && exc.getConcept().size() == 0 && exc.getFilter().size() == 0) {
211      excludeSystems.add(exc.getSystem());
212    }
213
214    if (exc.hasValueSet())
215      throw new Error("Processing Value set references in exclude is not yet done");
216    // importValueSet(imp.getValue(), params, profile);
217
218    CodeSystem cs = context.fetchCodeSystem(exc.getSystem());
219    if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(exc.getSystem())) {
220      excludeCodes(context.expandVS(exc, false), params);
221      return;
222    }
223
224    for (ConceptReferenceComponent c : exc.getConcept()) {
225      excludeCode(exc.getSystem(), c.getCode());
226    }
227
228    if (exc.getFilter().size() > 0)
229      throw new NotImplementedException("not done yet");
230  }
231
232  private void excludeCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params) {
233    for (ValueSetExpansionContainsComponent c : expand.getContains()) {
234      excludeCode(c.getSystem(), c.getCode());
235    }
236  }
237
238  private boolean existsInParams(List<ValueSetExpansionParameterComponent> params, String name, Type value) {
239    for (ValueSetExpansionParameterComponent p : params) {
240      if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false))
241        return true;
242    }
243    return false;
244  }
245
246  @Override
247  public ValueSetExpansionOutcome expand(ValueSet source, ExpansionProfile profile) {
248
249    if (profile == null)
250      profile = makeDefaultExpansion();
251    try {
252      focus = source.copy();
253      focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
254      focus.getExpansion().setTimestampElement(DateTimeType.now());
255      focus.getExpansion().setIdentifier(Factory.createUUID());
256      if (!profile.getUrl().startsWith("urn:uuid:"))
257        focus.getExpansion().addParameter().setName("profile").setValue(new UriType(profile.getUrl()));
258
259      if (source.hasCompose())
260        handleCompose(source.getCompose(), focus.getExpansion().getParameter(), profile);
261
262      if (canBeHeirarchy) {
263        for (ValueSetExpansionContainsComponent c : roots) {
264          focus.getExpansion().getContains().add(c);
265        }
266      } else {
267        for (ValueSetExpansionContainsComponent c : codes) {
268          if (map.containsKey(key(c)) && !c.getAbstract()) { // we may have added abstract codes earlier while we still thought it might be heirarchical, but later we gave up, so now ignore them
269            focus.getExpansion().getContains().add(c);
270            c.getContains().clear(); // make sure any heirarchy is wiped
271          }
272        }
273      }
274
275      if (total > 0) {
276        focus.getExpansion().setTotal(total);
277      }
278
279      return new ValueSetExpansionOutcome(focus);
280    } catch (NoTerminologyServiceException e) {
281      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
282      // that might fail too, but it might not, later.
283      return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage(), TerminologyServiceErrorClass.NOSERVICE);
284    } catch (RuntimeException e) {
285      // TODO: we should put something more specific instead of just Exception below, since
286      // it swallows bugs.. what would be expected to be caught there?
287      throw e;
288    } catch (Exception e) {
289      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
290      // that might fail too, but it might not, later.
291      return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage(), TerminologyServiceErrorClass.UNKNOWN);
292    }
293  }
294
295  private ExpansionProfile makeDefaultExpansion() {
296    ExpansionProfile res = new ExpansionProfile();
297    res.setUrl("urn:uuid:" + UUID.randomUUID().toString().toLowerCase());
298    res.setExcludeNested(true);
299    res.setIncludeDesignations(false);
300    return res;
301  }
302
303  private void addToHeirarchy(List<ValueSetExpansionContainsComponent> target, List<ValueSetExpansionContainsComponent> source) {
304    for (ValueSetExpansionContainsComponent s : source) {
305      target.add(s);
306    }
307  }
308
309  private String getCodeDisplay(CodeSystem cs, String code) throws TerminologyServiceException {
310    ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), code);
311    if (def == null)
312      throw new TerminologyServiceException("Unable to find code '" + code + "' in code system " + cs.getUrl());
313    return def.getDisplay();
314  }
315
316  private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> clist, String code) {
317    for (ConceptDefinitionComponent c : clist) {
318      if (code.equals(c.getCode()))
319        return c;
320      ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code);
321      if (v != null)
322        return v;
323    }
324    return null;
325  }
326
327  private void handleCompose(ValueSetComposeComponent compose, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile)
328    throws ETooCostly, FileNotFoundException, IOException, FHIRException {
329    // Exclude comes first because we build up a map of things to exclude
330    for (ConceptSetComponent inc : compose.getExclude())
331      excludeCodes(inc, params);
332    canBeHeirarchy = !profile.getExcludeNested() && excludeKeys.isEmpty() && excludeSystems.isEmpty();
333    boolean first = true;
334    for (ConceptSetComponent inc : compose.getInclude()) {
335      if (first == true)
336        first = false;
337      else
338        canBeHeirarchy = false;
339      includeCodes(inc, params, profile);
340    }
341
342  }
343
344  private ValueSet importValueSet(String value, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile)
345    throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException, FHIRFormatError {
346    if (value == null)
347      throw new TerminologyServiceException("unable to find value set with no identity");
348    ValueSet vs = context.fetchResource(ValueSet.class, value);
349    if (vs == null)
350      throw new TerminologyServiceException("Unable to find imported value set " + value);
351    ValueSetExpansionOutcome vso = factory.getExpander().expand(vs, profile);
352    if (vso.getError() != null)
353      throw new TerminologyServiceException("Unable to expand imported value set: " + vso.getError());
354    if (vso.getService() != null)
355      throw new TerminologyServiceException("Unable to expand imported value set " + value);
356    if (vs.hasVersion())
357      if (!existsInParams(params, "version", new UriType(vs.getUrl() + "|" + vs.getVersion())))
358        params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl() + "|" + vs.getVersion())));
359    for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
360      if (!existsInParams(params, p.getName(), p.getValue()))
361        params.add(p);
362    }
363    canBeHeirarchy = false; // if we're importing a value set, we have to be combining, so we won't try for a heirarchy
364    return vso.getValueset();
365  }
366
367  private void copyImportContains(List<ValueSetExpansionContainsComponent> list, ValueSetExpansionContainsComponent parent, ExpansionProfile profile, List<ValueSet> filter) {
368    for (ValueSetExpansionContainsComponent c : list) {
369      ValueSetExpansionContainsComponent np = addCode(c.getSystem(), c.getCode(), c.getDisplay(), parent, null, profile, c.getAbstract(), c.getInactive(), filter);
370      copyImportContains(c.getContains(), np, profile, filter);
371    }
372  }
373
374  private void includeCodes(ConceptSetComponent inc, List<ValueSetExpansionParameterComponent> params, ExpansionProfile profile) throws ETooCostly, FileNotFoundException, IOException, FHIRException {
375    List<ValueSet> imports = new ArrayList<ValueSet>();
376    for (UriType imp : inc.getValueSet())
377      imports.add(importValueSet(imp.getValue(), params, profile));
378
379    if (!inc.hasSystem()) {
380      if (imports.isEmpty()) // though this is not supposed to be the case
381        return;
382      ValueSet base = imports.get(0);
383      imports.remove(0);
384      copyImportContains(base.getExpansion().getContains(), null, profile, imports);
385    } else {
386      CodeSystem cs = context.fetchCodeSystem(inc.getSystem());
387      if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) {
388        addCodes(context.expandVS(inc, canBeHeirarchy), params, profile, imports);
389        return;
390      }
391
392      if (cs == null) {
393        if (context.isNoTerminologyServer())
394          throw new NoTerminologyServiceException("unable to find code system " + inc.getSystem().toString());
395        else
396          throw new TerminologyServiceException("unable to find code system " + inc.getSystem().toString());
397      }
398      if (cs.getContent() != CodeSystemContentMode.COMPLETE)
399        throw new TerminologyServiceException("Code system " + inc.getSystem().toString() + " is incomplete");
400      if (cs.hasVersion())
401        if (!existsInParams(params, "version", new UriType(cs.getUrl() + "|" + cs.getVersion())))
402          params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(cs.getUrl() + "|" + cs.getVersion())));
403
404      if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
405        // special case - add all the code system
406        for (ConceptDefinitionComponent def : cs.getConcept()) {
407          addCodeAndDescendents(cs, inc.getSystem(), def, null, profile, imports);
408        }
409      }
410
411      if (!inc.getConcept().isEmpty()) {
412        canBeHeirarchy = false;
413        for (ConceptReferenceComponent c : inc.getConcept()) {
414          addCode(inc.getSystem(), c.getCode(), Utilities.noString(c.getDisplay()) ? getCodeDisplay(cs, c.getCode()) : c.getDisplay(), null, convertDesignations(c.getDesignation()), profile, false,
415            CodeSystemUtilities.isInactive(cs, c.getCode()), imports);
416        }
417      }
418      if (inc.getFilter().size() > 1) {
419        canBeHeirarchy = false; // which will bt the case if we get around to supporting this
420        throw new TerminologyServiceException("Multiple filters not handled yet"); // need to and them, and this isn't done yet. But this shouldn't arise in non loinc and snomed value sets
421      }
422      if (inc.getFilter().size() == 1) {
423        ConceptSetFilterComponent fc = inc.getFilter().get(0);
424        if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) {
425          // special: all codes in the target code system under the value
426          ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
427          if (def == null)
428            throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
429          addCodeAndDescendents(cs, inc.getSystem(), def, null, profile, imports);
430        } else if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.DESCENDENTOF) {
431          // special: all codes in the target code system under the value
432          ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
433          if (def == null)
434            throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
435          for (ConceptDefinitionComponent c : def.getConcept())
436            addCodeAndDescendents(cs, inc.getSystem(), c, null, profile, imports);
437        } else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
438          // gg; note: wtf is this: if the filter is display=v, look up the code 'v', and see if it's diplsay is 'v'?
439          canBeHeirarchy = false;
440          ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
441          if (def != null) {
442            if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) {
443              if (def.getDisplay().contains(fc.getValue())) {
444                addCode(inc.getSystem(), def.getCode(), def.getDisplay(), null, def.getDesignation(), profile, CodeSystemUtilities.isNotSelectable(cs, def), CodeSystemUtilities.isInactive(cs, def),
445                  imports);
446              }
447            }
448          }
449        } else
450          throw new NotImplementedException("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet");
451      }
452    }
453  }
454
455  private List<ConceptDefinitionDesignationComponent> convertDesignations(List<ConceptReferenceDesignationComponent> list) {
456    List<ConceptDefinitionDesignationComponent> res = new ArrayList<CodeSystem.ConceptDefinitionDesignationComponent>();
457    for (ConceptReferenceDesignationComponent t : list) {
458      ConceptDefinitionDesignationComponent c = new ConceptDefinitionDesignationComponent();
459      c.setLanguage(t.getLanguage());
460      c.setUse(t.getUse());
461      c.setValue(t.getValue());
462    }
463    return res;
464  }
465
466  private String key(String uri, String code) {
467    return "{" + uri + "}" + code;
468  }
469
470  private String key(ValueSetExpansionContainsComponent c) {
471    return key(c.getSystem(), c.getCode());
472  }
473
474}