Source code

001package org.hl7.fhir.utilities.json;
002
003import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
004
005import java.io.File;
006
007/*
008  Copyright (c) 2011+, HL7, Inc.
009  All rights reserved.
010  
011  Redistribution and use in source and binary forms, with or without modification, 
012  are permitted provided that the following conditions are met:
013    
014   * Redistributions of source code must retain the above copyright notice, this 
015     list of conditions and the following disclaimer.
016   * Redistributions in binary form must reproduce the above copyright notice, 
017     this list of conditions and the following disclaimer in the documentation 
018     and/or other materials provided with the distribution.
019   * Neither the name of HL7 nor the names of its contributors may be used to 
020     endorse or promote products derived from this software without specific 
021     prior written permission.
022  
023  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
024  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
025  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
026  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
027  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
028  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
029  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
030  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
031  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
032  POSSIBILITY OF SUCH DAMAGE.
033  
034 */
035
036
037
038import java.io.IOException;
039import java.io.InputStream;
040import java.math.BigDecimal;
041import java.nio.charset.StandardCharsets;
042import java.util.Map;
043import java.util.Stack;
044
045import org.hl7.fhir.utilities.SimpleHTTPClient;
046import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
047import org.hl7.fhir.utilities.TextFile;
048import org.hl7.fhir.utilities.Utilities;
049
050import com.google.gson.Gson;
051import com.google.gson.GsonBuilder;
052import com.google.gson.JsonArray;
053import com.google.gson.JsonElement;
054import com.google.gson.JsonNull;
055import com.google.gson.JsonObject;
056import com.google.gson.JsonPrimitive;
057
058
059/**
060 * This is created to get a json parser that can track line numbers... grr...
061 * 
062 * @author Grahame Grieve
063 *
064 */
065public class JsonTrackingParser {
066
067        public class PresentedBigDecimal extends BigDecimal {
068
069          public String presentation;
070          
071    public PresentedBigDecimal(String value) {
072      super(value);
073      presentation = value;
074    }
075
076    public String getPresentation() {
077      return presentation;
078    }
079
080  }
081
082  public enum TokenType {
083                Open, Close, String, Number, Colon, Comma, OpenArray, CloseArray, Eof, Null, Boolean;
084        }
085        
086        public class LocationData {
087                private int line;
088                private int col;
089                
090                protected LocationData(int line, int col) {
091                        super();
092                        this.line = line;
093                        this.col = col;
094                }
095                
096                public int getLine() {
097                        return line;
098                }
099                
100                public int getCol() {
101                        return col;
102                }
103                
104                public void newLine() {
105                        line++;
106                        col = 1;                
107                }
108
109                public LocationData copy() {
110                        return new LocationData(line, col);
111                }
112        }
113        
114        private class State {
115                private String name;
116                private boolean isProp;
117                protected State(String name, boolean isProp) {
118                        super();
119                        this.name = name;
120                        this.isProp = isProp;
121                }
122                public String getName() {
123                        return name;
124                }
125                public boolean isProp() {
126                        return isProp;
127                }
128        }
129        
130        private class Lexer {
131                private String source;
132                private int cursor;
133                private String peek;
134                private String value;
135                private TokenType type;
136                private Stack<State> states = new Stack<State>();
137                private LocationData lastLocationBWS;
138                private LocationData lastLocationAWS;
139                private LocationData location;
140                private StringBuilder b = new StringBuilder();
141                
142    public Lexer(String source) throws IOException {
143        this.source = source;
144        cursor = -1;
145        location = new LocationData(1, 1);  
146        start();
147    }
148    
149    private boolean more() {
150        return peek != null || cursor < source.length(); 
151    }
152    
153    private String getNext(int length) throws IOException {
154        String result = "";
155      if (peek != null) {
156        if (peek.length() > length) {
157                result = peek.substring(0, length);
158                peek = peek.substring(length);
159        } else {
160                result = peek;
161                peek = null;
162        }
163      }
164      if (result.length() < length) {
165        int len = length - result.length(); 
166        if (cursor > source.length() - len) 
167                throw error("Attempt to read past end of source");
168        result = result + source.substring(cursor+1, cursor+len+1);
169        cursor = cursor + len;
170      }
171       for (char ch : result.toCharArray())
172        if (ch == '\n')
173          location.newLine();
174        else
175          location.col++;
176      return result;
177    }
178    
179    private char getNextChar() throws IOException {
180      if (peek != null) {
181        char ch = peek.charAt(0);
182        peek = peek.length() == 1 ? null : peek.substring(1);
183        return ch;
184      } else {
185        cursor++;
186        if (cursor >= source.length())
187          return (char) 0;
188        char ch = source.charAt(cursor);
189        if (ch == '\n') {
190          location.newLine();
191        } else {
192          location.col++;
193        }
194        return ch;
195      }
196    }
197    
198    private void push(char ch){
199        peek = peek == null ? String.valueOf(ch) : String.valueOf(ch)+peek;
200    }
201    
202    private void parseWord(String word, char ch, TokenType type) throws IOException {
203      this.type = type;
204      value = ""+ch+getNext(word.length()-1);
205      if (!value.equals(word))
206        throw error("Syntax error in json reading special word "+word);
207    }
208    
209    private IOException error(String msg) {
210      return new IOException("Error parsing JSON source: "+msg+" at Line "+Integer.toString(location.line)+" (path=["+path()+"])");
211    }
212    
213    private String path() {
214      if (states.empty())
215        return value;
216      else {
217        String result = "";
218        for (State s : states) 
219          result = result + '/'+ s.getName();
220        result = result + value;
221        return result;
222      }
223    }
224
225    public void start() throws IOException {
226//      char ch = getNextChar();
227//      if (ch = '\.uEF')
228//      begin
229//        // skip BOM
230//        getNextChar();
231//        getNextChar();
232//      end
233//      else
234//        push(ch);
235      next();
236    }
237    
238    public TokenType getType() {
239        return type;
240    }
241    
242    public String getValue() {
243        return value;
244    }
245
246
247    public LocationData getLastLocationBWS() {
248        return lastLocationBWS;
249    }
250
251    public LocationData getLastLocationAWS() {
252        return lastLocationAWS;
253    }
254
255    public void next() throws IOException {
256        lastLocationBWS = location.copy();
257        char ch;
258        do {
259                ch = getNextChar();
260                if (allowComments && ch == '/') {
261                  char ch1 = getNextChar();
262                  if (ch1 == '/') {
263                    while (more() && !Utilities.charInSet(ch, '\r', '\n')) {
264                      ch = getNextChar();
265                    }
266                  } else {
267                    push(ch1);
268                  }               
269                }
270        } while (more() && Utilities.charInSet(ch, ' ', '\r', '\n', '\t'));
271        lastLocationAWS = location.copy();
272
273        if (!more()) {
274                type = TokenType.Eof;
275        } else {
276                switch (ch) {
277                case '{' : 
278                        type = TokenType.Open;
279                        break;
280                case '}' : 
281                        type = TokenType.Close;
282                        break;
283                case '"' :
284                        type = TokenType.String;
285                        b.setLength(0);
286                        do {
287                                ch = getNextChar();
288                                if (ch == '\\') {
289                                        ch = getNextChar();
290                                        switch (ch) {
291              case '"': b.append('"'); break;
292              case '\'': b.append('\''); break;
293                                        case '\\': b.append('\\'); break;
294                                        case '/': b.append('/'); break;
295                                        case 'n': b.append('\n'); break;
296                                        case 'r': b.append('\r'); break;
297                                        case 't': b.append('\t'); break;
298                                        case 'u': b.append((char) Integer.parseInt(getNext(4), 16)); break;
299                                        default :
300                                                throw error("unknown escape sequence: \\"+ch);
301                                        }
302                                        ch = ' ';
303                                } else if (ch != '"')
304                                        b.append(ch);
305                        } while (more() && (ch != '"'));
306                        if (!more())
307                                throw error("premature termination of json stream during a string");
308                        value = b.toString();
309                        break;
310                case ':' : 
311                        type = TokenType.Colon;
312                        break;
313                case ',' : 
314                        type = TokenType.Comma;
315                        break;
316                case '[' : 
317                        type = TokenType.OpenArray;
318                        break;
319                case ']' : 
320                        type = TokenType.CloseArray;
321                        break;
322                case 't' : 
323                        parseWord("true", ch, TokenType.Boolean);
324                        break;
325                case 'f' : 
326                        parseWord("false", ch, TokenType.Boolean);
327                        break;
328                case 'n' : 
329                        parseWord("null", ch, TokenType.Null);
330                        break;
331                default:
332                        if ((ch >= '0' && ch <= '9') || ch == '-') {
333                                type = TokenType.Number;
334                                b.setLength(0);
335                                while (more() && ((ch >= '0' && ch <= '9') || ch == '-' || ch == '.') || ch == '+' || ch == 'e' || ch == 'E') {
336                                        b.append(ch);
337                                        ch = getNextChar();
338                                }
339                                value = b.toString();
340                                push(ch);
341                        } else
342                                throw error("Unexpected char '"+ch+"' in json stream");
343                }
344        }
345    }
346
347    public String consume(TokenType type) throws IOException {
348      if (this.type != type)
349        throw error("JSON syntax error - found "+this.type.toString()+" expecting "+type.toString());
350      String result = value;
351      next();
352      return result;
353    }
354
355        }
356
357        enum ItemType {
358          Object, String, Number, Boolean, Array, End, Eof, Null;
359        }
360        private Map<JsonElement, LocationData> map;
361  private Lexer lexer;
362  private ItemType itemType = ItemType.Object;
363  private String itemName;
364  private String itemValue;
365  private boolean errorOnDuplicates = true;
366  private boolean allowComments = false;
367
368  public static JsonObject parseJson(String source) throws IOException {
369    return parse(source, null);
370  }
371  
372  public static JsonObject parseJson(InputStream stream) throws IOException {
373    return parse(TextFile.streamToString(stream), null);
374  }
375  
376  public static JsonObject parseJson(byte[] stream) throws IOException {
377    return parse(TextFile.bytesToString(stream), null);
378  }
379  
380  public static JsonArray parseJsonArray(byte[] stream) throws IOException {
381    return parseArray(TextFile.bytesToString(stream), null);
382  }
383  
384  public static JsonObject parseJson(byte[] stream, boolean allowDuplicates) throws IOException {
385    return parse(TextFile.bytesToString(stream), null, allowDuplicates);
386  }
387  
388  public static JsonObject parseJson(File source) throws IOException {
389    return parse(TextFile.fileToString(source), null);
390  }
391  
392  public static JsonObject parseJsonFile(String source) throws IOException {
393    return parse(TextFile.fileToString(source), null);
394  }
395  
396  public static JsonObject parse(String source, Map<JsonElement, LocationData> map) throws IOException {
397    return parse(source, map, false);
398  }
399    
400  public static JsonArray parseArray(String source, Map<JsonElement, LocationData> map) throws IOException {
401    return parseArray(source, map, false);
402  }
403    
404    
405  public static JsonObject parse(String source, Map<JsonElement, LocationData> map, boolean allowDuplicates) throws IOException {
406    return parse(source, map, allowDuplicates, false);
407  }
408  
409  public static JsonArray parseArray(String source, Map<JsonElement, LocationData> map, boolean allowDuplicates) throws IOException {
410    return parseArray(source, map, allowDuplicates, false);
411  }
412  
413  public static JsonObject parse(String source, Map<JsonElement, LocationData> map, boolean allowDuplicates, boolean allowComments) throws IOException {
414    JsonTrackingParser self = new JsonTrackingParser();
415    self.map = map;
416    self.setErrorOnDuplicates(!allowDuplicates);
417    self.setAllowComments(allowComments);
418    return self.parse(Utilities.stripBOM(source));
419  }
420
421  public static JsonArray parseArray(String source, Map<JsonElement, LocationData> map, boolean allowDuplicates, boolean allowComments) throws IOException {
422    JsonTrackingParser self = new JsonTrackingParser();
423    self.map = map;
424    self.setErrorOnDuplicates(!allowDuplicates);
425    self.setAllowComments(allowComments);
426    return self.parseArray(Utilities.stripBOM(source));
427  }
428
429        private JsonObject parse(String source) throws IOException {
430                lexer = new Lexer(source);
431                JsonObject result = new JsonObject();
432                LocationData loc = lexer.location.copy();
433    if (lexer.getType() == TokenType.Open) {
434      lexer.next();
435      lexer.states.push(new State("", false));
436    } 
437    else
438      throw lexer.error("Unexpected content at start of JSON: "+lexer.getType().toString());
439
440    if (lexer.getType() != TokenType.Close) {
441      parseProperty();
442      readObject(result, true);
443    }
444    if (map != null)
445                  map.put(result, loc);
446    return result;
447        }
448
449  private JsonArray parseArray(String source) throws IOException {
450    return new Gson().fromJson(source, JsonArray.class);
451  }
452
453        private void readObject(JsonObject obj, boolean root) throws IOException {
454          if (map != null)
455      map.put(obj, lexer.location.copy());
456
457                while (!(itemType == ItemType.End) || (root && (itemType == ItemType.Eof))) {
458                        switch (itemType) {
459                        case Object:
460                                JsonObject child = new JsonObject(); //(obj.path+'.'+ItemName);
461                                LocationData loc = lexer.location.copy();
462              if (!obj.has(itemName))
463                obj.add(itemName, child);
464              else if (errorOnDuplicates)
465                throw lexer.error("Duplicated property name: "+itemName);
466                                next();
467                                readObject(child, false);
468                                if (map != null)
469                      map.put(obj, loc);
470                                break;
471                        case Boolean :
472                                JsonPrimitive v = new JsonPrimitive(Boolean.valueOf(itemValue));
473        if (!obj.has(itemName))
474                                obj.add(itemName, v);
475        else if (errorOnDuplicates)
476          throw lexer.error("Duplicated property name: "+itemName);
477                                if (map != null)
478                      map.put(v, lexer.location.copy());
479                                break;
480                        case String:
481                                v = new JsonPrimitive(itemValue);
482        if (!obj.has(itemName))
483                                obj.add(itemName, v);
484        else if (errorOnDuplicates)
485          throw lexer.error("Duplicated property name: "+itemName);
486                                if (map != null)
487                      map.put(v, lexer.location.copy());
488                                break;
489                        case Number:
490                                v = new JsonPrimitive(new PresentedBigDecimal(itemValue));
491        if (!obj.has(itemName))
492                                obj.add(itemName, v);
493        else if (errorOnDuplicates)
494          throw lexer.error("Duplicated property name: "+itemName);
495                                if (map != null)
496                      map.put(v, lexer.location.copy());
497                                break;
498                        case Null:
499                                JsonNull n = new JsonNull();
500        if (!obj.has(itemName))
501                                obj.add(itemName, n);
502        else if (errorOnDuplicates)
503          throw lexer.error("Duplicated property name: "+itemName);
504                                if (map != null)
505                      map.put(n, lexer.location.copy());
506                                break;
507                        case Array:
508                                JsonArray arr = new JsonArray(); // (obj.path+'.'+ItemName);
509                                loc = lexer.location.copy();
510        if (!obj.has(itemName))
511                                obj.add(itemName, arr);
512        else if (errorOnDuplicates)
513          throw lexer.error("Duplicated property name: "+itemName);
514                                next();
515                                if (!readArray(arr, false))
516                                  next(true);
517                                if (map != null)
518                      map.put(arr, loc);
519                                break;
520                        case Eof : 
521                                throw lexer.error("Unexpected End of File");
522                        case End:
523                                // TODO GG: This isn't handled. Should it be?
524                                break;
525                        }
526                        next();
527                }
528        }
529
530        private boolean readArray(JsonArray arr, boolean root) throws IOException {
531          boolean res = false;
532          while (!((itemType == ItemType.End) || (root && (itemType == ItemType.Eof)))) {
533            res  = true;
534            switch (itemType) {
535            case Object:
536                JsonObject obj  = new JsonObject(); // (arr.path+'['+inttostr(i)+']');
537                                LocationData loc = lexer.location.copy();
538                arr.add(obj);
539              next();
540              readObject(obj, false);
541              if (map != null)
542                map.put(obj, loc);
543              break;
544            case String:
545                JsonPrimitive v = new JsonPrimitive(itemValue);
546                                arr.add(v);
547                                if (map != null)
548                      map.put(v, lexer.location.copy());
549                                break;
550            case Number:
551                v = new JsonPrimitive(new BigDecimal(itemValue));
552                                arr.add(v);
553                                if (map != null)
554                      map.put(v, lexer.location.copy());
555                                break;
556            case Null :
557                JsonNull n = new JsonNull();
558                                arr.add(n);
559                                if (map != null)
560                      map.put(n, lexer.location.copy());
561                                break;
562            case Array:
563        JsonArray child = new JsonArray(); // (arr.path+'['+inttostr(i)+']');
564                                loc = lexer.location.copy();
565                                arr.add(child);
566        next();
567              readArray(child, false);
568              if (map != null)
569                map.put(arr, loc);
570        break;
571            case Eof : 
572                throw lexer.error("Unexpected End of File");
573                 case End:
574                 case Boolean:
575                        // TODO GG: These aren't handled. SHould they be?
576                        break;
577            }
578            next();
579          }
580          return res;
581        }
582
583  private void next() throws IOException {
584    next(false);
585  }
586  
587        private void next(boolean noPop) throws IOException {
588                switch (itemType) {
589                case Object :
590                        lexer.consume(TokenType.Open);
591                        lexer.states.push(new State(itemName, false));
592                        if (lexer.getType() == TokenType.Close) {
593                                itemType = ItemType.End;
594                                lexer.next();
595                        } else
596                                parseProperty();
597                        break;
598                case Null:
599                case String:
600                case Number: 
601                case End: 
602                case Boolean :
603                        if (itemType == ItemType.End && !noPop)
604                                lexer.states.pop();
605                        if (lexer.getType() == TokenType.Comma) {
606                                lexer.next();
607                                parseProperty();
608                        } else if (lexer.getType() == TokenType.Close) {
609                                itemType = ItemType.End;
610                                lexer.next();
611                        } else if (lexer.getType() == TokenType.CloseArray) {
612                                itemType = ItemType.End;
613                                lexer.next();
614                        } else if (lexer.getType() == TokenType.Eof) {
615                                itemType = ItemType.Eof;
616                        } else
617                                throw lexer.error("Unexpected JSON syntax");
618                        break;
619                case Array :
620                        lexer.next();
621                        lexer.states.push(new State(itemName+"[]", true));
622                        parseProperty();
623                        break;
624                case Eof :
625                        throw lexer.error("JSON Syntax Error - attempt to read past end of json stream");
626                default:
627                        throw lexer.error("not done yet (a): "+itemType.toString());
628                }
629        }
630
631        private void parseProperty() throws IOException {
632                if (!lexer.states.peek().isProp) {
633                        itemName = lexer.consume(TokenType.String);
634                        itemValue = null;
635                        lexer.consume(TokenType.Colon);
636                }
637                switch (lexer.getType()) {
638                case Null :
639                        itemType = ItemType.Null;
640                        itemValue = lexer.value;
641                        lexer.next();
642                        break;
643                case String :
644                        itemType = ItemType.String;
645                        itemValue = lexer.value;
646                        lexer.next();
647                        break;
648                case Boolean :
649                        itemType = ItemType.Boolean;
650                        itemValue = lexer.value;
651                        lexer.next();
652                        break;
653                case Number :
654                        itemType = ItemType.Number;
655                        itemValue = lexer.value;
656                        lexer.next();
657                        break;
658                case Open :
659                        itemType = ItemType.Object;
660                        break;
661                case OpenArray :
662                        itemType = ItemType.Array;
663                        break;
664                case CloseArray :
665                        itemType = ItemType.End;
666                        break;
667                        // case Close, , case Colon, case Comma, case OpenArray,       !
668                default:
669                        throw lexer.error("not done yet (b): "+lexer.getType().toString());
670                }
671        }
672
673  public boolean isErrorOnDuplicates() {
674    return errorOnDuplicates;
675  }
676
677  public void setErrorOnDuplicates(boolean errorOnDuplicates) {
678    this.errorOnDuplicates = errorOnDuplicates;
679  }
680
681  
682  public boolean isAllowComments() {
683    return allowComments;
684  }
685
686  public void setAllowComments(boolean allowComments) {
687    this.allowComments = allowComments;
688  }
689
690  public static void write(JsonObject json, File file) throws IOException {
691    Gson gson = new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create();
692    String jcnt = gson.toJson(json);
693    TextFile.stringToFile(jcnt, file);    
694  }
695    
696  public static void write(JsonObject json, File file, boolean pretty) throws IOException {
697    Gson gson = pretty ? new GsonBuilder().setPrettyPrinting().create() : new GsonBuilder().create();
698    String jcnt = gson.toJson(json);
699    TextFile.stringToFile(jcnt, file);    
700  }
701    
702  public static void write(JsonObject json, String fileName) throws IOException {
703    Gson gson = new GsonBuilder().setPrettyPrinting().create();
704    String jcnt = gson.toJson(json);
705    TextFile.stringToFile(jcnt, fileName);    
706  }
707  
708  public static String write(JsonObject json) {
709    Gson gson = new GsonBuilder().setPrettyPrinting().create();
710    return gson.toJson(json);    
711  }
712
713  public static String writeDense(JsonObject json) {
714    Gson gson = new GsonBuilder().create();
715    return gson.toJson(json);    
716  }
717
718  public static byte[] writeBytes(JsonObject json, boolean pretty) {
719    if (pretty) {
720      Gson gson = new GsonBuilder().setPrettyPrinting().create();
721      return gson.toJson(json).getBytes(StandardCharsets.UTF_8);    
722    } else {
723      Gson gson = new GsonBuilder().create();
724      return gson.toJson(json).getBytes(StandardCharsets.UTF_8);    
725    }    
726  }
727  
728  public static JsonObject fetchJson(String source) throws IOException {
729    SimpleHTTPClient fetcher = new SimpleHTTPClient();
730    HTTPResult res = fetcher.get(source+"?nocache=" + System.currentTimeMillis());
731    res.checkThrowException();
732    return parseJson(res.getContent());
733  }
734  
735  public static JsonArray fetchJsonArray(String source) throws IOException {
736    SimpleHTTPClient fetcher = new SimpleHTTPClient();
737    HTTPResult res = fetcher.get(source+"?nocache=" + System.currentTimeMillis());
738    res.checkThrowException();
739    return parseJsonArray(res.getContent());
740  }
741        
742}