001// License: GPL. See LICENSE file for details.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.InputStream;
007import java.io.InputStreamReader;
008import java.text.MessageFormat;
009import java.util.ArrayList;
010import java.util.Collection;
011import java.util.List;
012import java.util.regex.Matcher;
013import java.util.regex.Pattern;
014
015import javax.xml.stream.Location;
016import javax.xml.stream.XMLInputFactory;
017import javax.xml.stream.XMLStreamConstants;
018import javax.xml.stream.XMLStreamException;
019import javax.xml.stream.XMLStreamReader;
020
021import org.openstreetmap.josm.Main;
022import org.openstreetmap.josm.data.Bounds;
023import org.openstreetmap.josm.data.coor.LatLon;
024import org.openstreetmap.josm.data.osm.Changeset;
025import org.openstreetmap.josm.data.osm.DataSet;
026import org.openstreetmap.josm.data.osm.DataSource;
027import org.openstreetmap.josm.data.osm.Node;
028import org.openstreetmap.josm.data.osm.NodeData;
029import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
030import org.openstreetmap.josm.data.osm.PrimitiveData;
031import org.openstreetmap.josm.data.osm.Relation;
032import org.openstreetmap.josm.data.osm.RelationData;
033import org.openstreetmap.josm.data.osm.RelationMemberData;
034import org.openstreetmap.josm.data.osm.Tagged;
035import org.openstreetmap.josm.data.osm.User;
036import org.openstreetmap.josm.data.osm.Way;
037import org.openstreetmap.josm.data.osm.WayData;
038import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
039import org.openstreetmap.josm.gui.progress.ProgressMonitor;
040import org.openstreetmap.josm.tools.CheckParameterUtil;
041import org.openstreetmap.josm.tools.DateUtils;
042
043/**
044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
045 *
046 * For each xml element, there is a dedicated method.
047 * The XMLStreamReader cursor points to the start of the element, when the method is
048 * entered, and it must point to the end of the same element, when it is exited.
049 */
050public class OsmReader extends AbstractReader {
051
052    protected XMLStreamReader parser;
053
054    protected boolean cancel;
055
056    /** Used by plugins to register themselves as data postprocessors. */
057    public static List<OsmServerReadPostprocessor> postprocessors;
058
059    /** register a new postprocessor */
060    public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
061        if (postprocessors == null) {
062            postprocessors = new ArrayList<OsmServerReadPostprocessor>();
063        }
064        postprocessors.add(pp);
065    }
066
067    /** deregister a postprocessor previously registered with registerPostprocessor */
068    public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
069        if (postprocessors != null) {
070            postprocessors.remove(pp);
071        }
072    }
073
074    /**
075     * constructor (for private and subclasses use only)
076     *
077     * @see #parseDataSet(InputStream, ProgressMonitor)
078     */
079    protected OsmReader() {
080    }
081
082    protected void setParser(XMLStreamReader parser) {
083        this.parser = parser;
084    }
085
086    protected void throwException(String msg) throws XMLStreamException {
087        throw new OsmParsingException(msg, parser.getLocation());
088    }
089
090    protected void parse() throws XMLStreamException {
091        int event = parser.getEventType();
092        while (true) {
093            if (event == XMLStreamConstants.START_ELEMENT) {
094                parseRoot();
095            } else if (event == XMLStreamConstants.END_ELEMENT)
096                return;
097            if (parser.hasNext()) {
098                event = parser.next();
099            } else {
100                break;
101            }
102        }
103        parser.close();
104    }
105
106    protected void parseRoot() throws XMLStreamException {
107        if (parser.getLocalName().equals("osm")) {
108            parseOsm();
109        } else {
110            parseUnknown();
111        }
112    }
113
114    private void parseOsm() throws XMLStreamException {
115        String v = parser.getAttributeValue(null, "version");
116        if (v == null) {
117            throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
118        }
119        if (!(v.equals("0.5") || v.equals("0.6"))) {
120            throwException(tr("Unsupported version: {0}", v));
121        }
122        ds.setVersion(v);
123        String upload = parser.getAttributeValue(null, "upload");
124        if (upload != null) {
125            ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
126        }
127        String generator = parser.getAttributeValue(null, "generator");
128        Long uploadChangesetId = null;
129        if (parser.getAttributeValue(null, "upload-changeset") != null) {
130            uploadChangesetId = getLong("upload-changeset");
131        }
132        while (true) {
133            int event = parser.next();
134
135            if (cancel) {
136                cancel = false;
137                throwException(tr("Reading was canceled"));
138            }
139
140            if (event == XMLStreamConstants.START_ELEMENT) {
141                if (parser.getLocalName().equals("bounds")) {
142                    parseBounds(generator);
143                } else if (parser.getLocalName().equals("node")) {
144                    parseNode();
145                } else if (parser.getLocalName().equals("way")) {
146                    parseWay();
147                } else if (parser.getLocalName().equals("relation")) {
148                    parseRelation();
149                } else if (parser.getLocalName().equals("changeset")) {
150                    parseChangeset(uploadChangesetId);
151                } else {
152                    parseUnknown();
153                }
154            } else if (event == XMLStreamConstants.END_ELEMENT)
155                return;
156        }
157    }
158
159    private void parseBounds(String generator) throws XMLStreamException {
160        String minlon = parser.getAttributeValue(null, "minlon");
161        String minlat = parser.getAttributeValue(null, "minlat");
162        String maxlon = parser.getAttributeValue(null, "maxlon");
163        String maxlat = parser.getAttributeValue(null, "maxlat");
164        String origin = parser.getAttributeValue(null, "origin");
165        if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
166            if (origin == null) {
167                origin = generator;
168            }
169            Bounds bounds = new Bounds(
170                    Double.parseDouble(minlat), Double.parseDouble(minlon),
171                    Double.parseDouble(maxlat), Double.parseDouble(maxlon));
172            if (bounds.isOutOfTheWorld()) {
173                Bounds copy = new Bounds(bounds);
174                bounds.normalize();
175                Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
176            }
177            DataSource src = new DataSource(bounds, origin);
178            ds.dataSources.add(src);
179        } else {
180            throwException(tr(
181                    "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
182                    minlon, minlat, maxlon, maxlat, origin
183            ));
184        }
185        jumpToEnd();
186    }
187
188    protected Node parseNode() throws XMLStreamException {
189        NodeData nd = new NodeData();
190        String lat = parser.getAttributeValue(null, "lat");
191        String lon = parser.getAttributeValue(null, "lon");
192        if (lat != null && lon != null) {
193            nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
194        }
195        readCommon(nd);
196        Node n = new Node(nd.getId(), nd.getVersion());
197        n.setVisible(nd.isVisible());
198        n.load(nd);
199        externalIdMap.put(nd.getPrimitiveId(), n);
200        while (true) {
201            int event = parser.next();
202            if (event == XMLStreamConstants.START_ELEMENT) {
203                if (parser.getLocalName().equals("tag")) {
204                    parseTag(n);
205                } else {
206                    parseUnknown();
207                }
208            } else if (event == XMLStreamConstants.END_ELEMENT)
209                return n;
210        }
211    }
212
213    protected Way parseWay() throws XMLStreamException {
214        WayData wd = new WayData();
215        readCommon(wd);
216        Way w = new Way(wd.getId(), wd.getVersion());
217        w.setVisible(wd.isVisible());
218        w.load(wd);
219        externalIdMap.put(wd.getPrimitiveId(), w);
220
221        Collection<Long> nodeIds = new ArrayList<Long>();
222        while (true) {
223            int event = parser.next();
224            if (event == XMLStreamConstants.START_ELEMENT) {
225                if (parser.getLocalName().equals("nd")) {
226                    nodeIds.add(parseWayNode(w));
227                } else if (parser.getLocalName().equals("tag")) {
228                    parseTag(w);
229                } else {
230                    parseUnknown();
231                }
232            } else if (event == XMLStreamConstants.END_ELEMENT) {
233                break;
234            }
235        }
236        if (w.isDeleted() && !nodeIds.isEmpty()) {
237            Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId()));
238            nodeIds = new ArrayList<Long>();
239        }
240        ways.put(wd.getUniqueId(), nodeIds);
241        return w;
242    }
243
244    private long parseWayNode(Way w) throws XMLStreamException {
245        if (parser.getAttributeValue(null, "ref") == null) {
246            throwException(
247                    tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
248            );
249        }
250        long id = getLong("ref");
251        if (id == 0) {
252            throwException(
253                    tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
254            );
255        }
256        jumpToEnd();
257        return id;
258    }
259
260    protected Relation parseRelation() throws XMLStreamException {
261        RelationData rd = new RelationData();
262        readCommon(rd);
263        Relation r = new Relation(rd.getId(), rd.getVersion());
264        r.setVisible(rd.isVisible());
265        r.load(rd);
266        externalIdMap.put(rd.getPrimitiveId(), r);
267
268        Collection<RelationMemberData> members = new ArrayList<RelationMemberData>();
269        while (true) {
270            int event = parser.next();
271            if (event == XMLStreamConstants.START_ELEMENT) {
272                if (parser.getLocalName().equals("member")) {
273                    members.add(parseRelationMember(r));
274                } else if (parser.getLocalName().equals("tag")) {
275                    parseTag(r);
276                } else {
277                    parseUnknown();
278                }
279            } else if (event == XMLStreamConstants.END_ELEMENT) {
280                break;
281            }
282        }
283        if (r.isDeleted() && !members.isEmpty()) {
284            Main.info(tr("Deleted relation {0} contains members", r.getUniqueId()));
285            members = new ArrayList<RelationMemberData>();
286        }
287        relations.put(rd.getUniqueId(), members);
288        return r;
289    }
290
291    private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
292        String role = null;
293        OsmPrimitiveType type = null;
294        long id = 0;
295        String value = parser.getAttributeValue(null, "ref");
296        if (value == null) {
297            throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId()));
298        }
299        try {
300            id = Long.parseLong(value);
301        } catch(NumberFormatException e) {
302            throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value));
303        }
304        value = parser.getAttributeValue(null, "type");
305        if (value == null) {
306            throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
307        }
308        try {
309            type = OsmPrimitiveType.fromApiTypeName(value);
310        } catch(IllegalArgumentException e) {
311            throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value));
312        }
313        value = parser.getAttributeValue(null, "role");
314        role = value;
315
316        if (id == 0) {
317            throwException(tr("Incomplete <member> specification with ref=0"));
318        }
319        jumpToEnd();
320        return new RelationMemberData(role, type, id);
321    }
322
323    private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
324
325        Long id = null;
326        if (parser.getAttributeValue(null, "id") != null) {
327            id = getLong("id");
328        }
329        // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
330        if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) {
331            uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
332            while (true) {
333                int event = parser.next();
334                if (event == XMLStreamConstants.START_ELEMENT) {
335                    if (parser.getLocalName().equals("tag")) {
336                        parseTag(uploadChangeset);
337                    } else {
338                        parseUnknown();
339                    }
340                } else if (event == XMLStreamConstants.END_ELEMENT)
341                    return;
342            }
343        } else {
344            jumpToEnd(false);
345        }
346    }
347
348    private void parseTag(Tagged t) throws XMLStreamException {
349        String key = parser.getAttributeValue(null, "k");
350        String value = parser.getAttributeValue(null, "v");
351        if (key == null || value == null) {
352            throwException(tr("Missing key or value attribute in tag."));
353        }
354        t.put(key.intern(), value.intern());
355        jumpToEnd();
356    }
357
358    protected void parseUnknown(boolean printWarning) throws XMLStreamException {
359        if (printWarning) {
360            Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName()));
361        }
362        while (true) {
363            int event = parser.next();
364            if (event == XMLStreamConstants.START_ELEMENT) {
365                parseUnknown(false); /* no more warning for inner elements */
366            } else if (event == XMLStreamConstants.END_ELEMENT)
367                return;
368        }
369    }
370
371    protected void parseUnknown() throws XMLStreamException {
372        parseUnknown(true);
373    }
374
375    /**
376     * When cursor is at the start of an element, moves it to the end tag of that element.
377     * Nested content is skipped.
378     *
379     * This is basically the same code as parseUnknown(), except for the warnings, which
380     * are displayed for inner elements and not at top level.
381     */
382    private void jumpToEnd(boolean printWarning) throws XMLStreamException {
383        while (true) {
384            int event = parser.next();
385            if (event == XMLStreamConstants.START_ELEMENT) {
386                parseUnknown(printWarning);
387            } else if (event == XMLStreamConstants.END_ELEMENT)
388                return;
389        }
390    }
391
392    private void jumpToEnd() throws XMLStreamException {
393        jumpToEnd(true);
394    }
395
396    private User createUser(String uid, String name) throws XMLStreamException {
397        if (uid == null) {
398            if (name == null)
399                return null;
400            return User.createLocalUser(name);
401        }
402        try {
403            long id = Long.parseLong(uid);
404            return User.createOsmUser(id, name);
405        } catch(NumberFormatException e) {
406            throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid));
407        }
408        return null;
409    }
410
411    /**
412     * Read out the common attributes and put them into current OsmPrimitive.
413     */
414    private void readCommon(PrimitiveData current) throws XMLStreamException {
415        current.setId(getLong("id"));
416        if (current.getUniqueId() == 0) {
417            throwException(tr("Illegal object with ID=0."));
418        }
419
420        String time = parser.getAttributeValue(null, "timestamp");
421        if (time != null && time.length() != 0) {
422            current.setTimestamp(DateUtils.fromString(time));
423        }
424
425        // user attribute added in 0.4 API
426        String user = parser.getAttributeValue(null, "user");
427        // uid attribute added in 0.6 API
428        String uid = parser.getAttributeValue(null, "uid");
429        current.setUser(createUser(uid, user));
430
431        // visible attribute added in 0.4 API
432        String visible = parser.getAttributeValue(null, "visible");
433        if (visible != null) {
434            current.setVisible(Boolean.parseBoolean(visible));
435        }
436
437        String versionString = parser.getAttributeValue(null, "version");
438        int version = 0;
439        if (versionString != null) {
440            try {
441                version = Integer.parseInt(versionString);
442            } catch(NumberFormatException e) {
443                throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
444            }
445            if (ds.getVersion().equals("0.6")){
446                if (version <= 0 && current.getUniqueId() > 0) {
447                    throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString));
448                } else if (version < 0 && current.getUniqueId() <= 0) {
449                    Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6"));
450                    version = 0;
451                }
452            } else if (ds.getVersion().equals("0.5")) {
453                if (version <= 0 && current.getUniqueId() > 0) {
454                    Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
455                    version = 1;
456                } else if (version < 0 && current.getUniqueId() <= 0) {
457                    Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5"));
458                    version = 0;
459                }
460            } else {
461                // should not happen. API version has been checked before
462                throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
463            }
464        } else {
465            // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
466            //
467            if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) {
468                throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
469            } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
470                // default version in 0.5 files for existing primitives
471                Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5"));
472                version= 1;
473            } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) {
474                // default version in 0.5 files for new primitives, no warning necessary. This is
475                // (was) legal in API 0.5
476                version= 0;
477            }
478        }
479        current.setVersion(version);
480
481        String action = parser.getAttributeValue(null, "action");
482        if (action == null) {
483            // do nothing
484        } else if (action.equals("delete")) {
485            current.setDeleted(true);
486            current.setModified(current.isVisible());
487        } else if (action.equals("modify")) {
488            current.setModified(true);
489        }
490
491        String v = parser.getAttributeValue(null, "changeset");
492        if (v == null) {
493            current.setChangesetId(0);
494        } else {
495            try {
496                current.setChangesetId(Integer.parseInt(v));
497            } catch(NumberFormatException e) {
498                if (current.getUniqueId() <= 0) {
499                    // for a new primitive we just log a warning
500                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
501                    current.setChangesetId(0);
502                } else {
503                    // for an existing primitive this is a problem
504                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
505                }
506            }
507            if (current.getChangesetId() <=0) {
508                if (current.getUniqueId() <= 0) {
509                    // for a new primitive we just log a warning
510                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId()));
511                    current.setChangesetId(0);
512                } else {
513                    // for an existing primitive this is a problem
514                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
515                }
516            }
517        }
518    }
519
520    private long getLong(String name) throws XMLStreamException {
521        String value = parser.getAttributeValue(null, name);
522        if (value == null) {
523            throwException(tr("Missing required attribute ''{0}''.",name));
524        }
525        try {
526            return Long.parseLong(value);
527        } catch(NumberFormatException e) {
528            throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value));
529        }
530        return 0; // should not happen
531    }
532
533    private static class OsmParsingException extends XMLStreamException {
534        public OsmParsingException() {
535            super();
536        }
537
538        public OsmParsingException(String msg) {
539            super(msg);
540        }
541
542        public OsmParsingException(String msg, Location location) {
543            super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
544            this.location = location;
545        }
546
547        public OsmParsingException(String msg, Location location, Throwable th) {
548            super(msg, th);
549            this.location = location;
550        }
551
552        public OsmParsingException(String msg, Throwable th) {
553            super(msg, th);
554        }
555
556        public OsmParsingException(Throwable th) {
557            super(th);
558        }
559
560        @Override
561        public String getMessage() {
562            String msg = super.getMessage();
563            if (msg == null) {
564                msg = getClass().getName();
565            }
566            if (getLocation() == null)
567                return msg;
568            msg = msg + " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
569            return msg;
570        }
571    }
572
573    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
574        if (progressMonitor == null) {
575            progressMonitor = NullProgressMonitor.INSTANCE;
576        }
577        ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() {
578            @Override public void operationCanceled() {
579                cancel = true;
580            }
581        };
582        progressMonitor.addCancelListener(cancelListener);
583        CheckParameterUtil.ensureParameterNotNull(source, "source");
584        try {
585            progressMonitor.beginTask(tr("Prepare OSM data...", 2));
586            progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
587
588            InputStreamReader ir = UTFInputStreamReader.create(source, "UTF-8");
589            XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
590            setParser(parser);
591            parse();
592            progressMonitor.worked(1);
593
594            progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
595            prepareDataSet();
596            progressMonitor.worked(1);
597
598            // iterate over registered postprocessors and give them each a chance
599            // to modify the dataset we have just loaded.
600            if (postprocessors != null) {
601                for (OsmServerReadPostprocessor pp : postprocessors) {
602                    pp.postprocessDataSet(getDataSet(), progressMonitor);
603                }
604            }
605            return getDataSet();
606        } catch(IllegalDataException e) {
607            throw e;
608        } catch(OsmParsingException e) {
609            throw new IllegalDataException(e.getMessage(), e);
610        } catch(XMLStreamException e) {
611            String msg = e.getMessage();
612            Pattern p = Pattern.compile("Message: (.+)");
613            Matcher m = p.matcher(msg);
614            if (m.find()) {
615                msg = m.group(1);
616            }
617            if (e.getLocation() != null)
618                throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
619            else
620                throw new IllegalDataException(msg, e);
621        } catch(Exception e) {
622            throw new IllegalDataException(e);
623        } finally {
624            progressMonitor.finishTask();
625            progressMonitor.removeCancelListener(cancelListener);
626        }
627    }
628
629    /**
630     * Parse the given input source and return the dataset.
631     *
632     * @param source the source input stream. Must not be null.
633     * @param progressMonitor  the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
634     *
635     * @return the dataset with the parsed data
636     * @throws IllegalDataException thrown if the an error was found while parsing the data from the source
637     * @throws IllegalArgumentException thrown if source is null
638     */
639    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
640        return new OsmReader().doParseDataSet(source, progressMonitor);
641    }
642}