001// License: GPL. See LICENSE file for details. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.InputStream; 007import java.io.InputStreamReader; 008import java.text.MessageFormat; 009import java.util.ArrayList; 010import java.util.Collection; 011import java.util.List; 012import java.util.regex.Matcher; 013import java.util.regex.Pattern; 014 015import javax.xml.stream.Location; 016import javax.xml.stream.XMLInputFactory; 017import javax.xml.stream.XMLStreamConstants; 018import javax.xml.stream.XMLStreamException; 019import javax.xml.stream.XMLStreamReader; 020 021import org.openstreetmap.josm.Main; 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.coor.LatLon; 024import org.openstreetmap.josm.data.osm.Changeset; 025import org.openstreetmap.josm.data.osm.DataSet; 026import org.openstreetmap.josm.data.osm.DataSource; 027import org.openstreetmap.josm.data.osm.Node; 028import org.openstreetmap.josm.data.osm.NodeData; 029import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 030import org.openstreetmap.josm.data.osm.PrimitiveData; 031import org.openstreetmap.josm.data.osm.Relation; 032import org.openstreetmap.josm.data.osm.RelationData; 033import org.openstreetmap.josm.data.osm.RelationMemberData; 034import org.openstreetmap.josm.data.osm.Tagged; 035import org.openstreetmap.josm.data.osm.User; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.data.osm.WayData; 038import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 039import org.openstreetmap.josm.gui.progress.ProgressMonitor; 040import org.openstreetmap.josm.tools.CheckParameterUtil; 041import org.openstreetmap.josm.tools.DateUtils; 042 043/** 044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it. 045 * 046 * For each xml element, there is a dedicated method. 047 * The XMLStreamReader cursor points to the start of the element, when the method is 048 * entered, and it must point to the end of the same element, when it is exited. 049 */ 050public class OsmReader extends AbstractReader { 051 052 protected XMLStreamReader parser; 053 054 protected boolean cancel; 055 056 /** Used by plugins to register themselves as data postprocessors. */ 057 public static List<OsmServerReadPostprocessor> postprocessors; 058 059 /** register a new postprocessor */ 060 public static void registerPostprocessor(OsmServerReadPostprocessor pp) { 061 if (postprocessors == null) { 062 postprocessors = new ArrayList<OsmServerReadPostprocessor>(); 063 } 064 postprocessors.add(pp); 065 } 066 067 /** deregister a postprocessor previously registered with registerPostprocessor */ 068 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { 069 if (postprocessors != null) { 070 postprocessors.remove(pp); 071 } 072 } 073 074 /** 075 * constructor (for private and subclasses use only) 076 * 077 * @see #parseDataSet(InputStream, ProgressMonitor) 078 */ 079 protected OsmReader() { 080 } 081 082 protected void setParser(XMLStreamReader parser) { 083 this.parser = parser; 084 } 085 086 protected void throwException(String msg) throws XMLStreamException { 087 throw new OsmParsingException(msg, parser.getLocation()); 088 } 089 090 protected void parse() throws XMLStreamException { 091 int event = parser.getEventType(); 092 while (true) { 093 if (event == XMLStreamConstants.START_ELEMENT) { 094 parseRoot(); 095 } else if (event == XMLStreamConstants.END_ELEMENT) 096 return; 097 if (parser.hasNext()) { 098 event = parser.next(); 099 } else { 100 break; 101 } 102 } 103 parser.close(); 104 } 105 106 protected void parseRoot() throws XMLStreamException { 107 if (parser.getLocalName().equals("osm")) { 108 parseOsm(); 109 } else { 110 parseUnknown(); 111 } 112 } 113 114 private void parseOsm() throws XMLStreamException { 115 String v = parser.getAttributeValue(null, "version"); 116 if (v == null) { 117 throwException(tr("Missing mandatory attribute ''{0}''.", "version")); 118 } 119 if (!(v.equals("0.5") || v.equals("0.6"))) { 120 throwException(tr("Unsupported version: {0}", v)); 121 } 122 ds.setVersion(v); 123 String upload = parser.getAttributeValue(null, "upload"); 124 if (upload != null) { 125 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload)); 126 } 127 String generator = parser.getAttributeValue(null, "generator"); 128 Long uploadChangesetId = null; 129 if (parser.getAttributeValue(null, "upload-changeset") != null) { 130 uploadChangesetId = getLong("upload-changeset"); 131 } 132 while (true) { 133 int event = parser.next(); 134 135 if (cancel) { 136 cancel = false; 137 throwException(tr("Reading was canceled")); 138 } 139 140 if (event == XMLStreamConstants.START_ELEMENT) { 141 if (parser.getLocalName().equals("bounds")) { 142 parseBounds(generator); 143 } else if (parser.getLocalName().equals("node")) { 144 parseNode(); 145 } else if (parser.getLocalName().equals("way")) { 146 parseWay(); 147 } else if (parser.getLocalName().equals("relation")) { 148 parseRelation(); 149 } else if (parser.getLocalName().equals("changeset")) { 150 parseChangeset(uploadChangesetId); 151 } else { 152 parseUnknown(); 153 } 154 } else if (event == XMLStreamConstants.END_ELEMENT) 155 return; 156 } 157 } 158 159 private void parseBounds(String generator) throws XMLStreamException { 160 String minlon = parser.getAttributeValue(null, "minlon"); 161 String minlat = parser.getAttributeValue(null, "minlat"); 162 String maxlon = parser.getAttributeValue(null, "maxlon"); 163 String maxlat = parser.getAttributeValue(null, "maxlat"); 164 String origin = parser.getAttributeValue(null, "origin"); 165 if (minlon != null && maxlon != null && minlat != null && maxlat != null) { 166 if (origin == null) { 167 origin = generator; 168 } 169 Bounds bounds = new Bounds( 170 Double.parseDouble(minlat), Double.parseDouble(minlon), 171 Double.parseDouble(maxlat), Double.parseDouble(maxlon)); 172 if (bounds.isOutOfTheWorld()) { 173 Bounds copy = new Bounds(bounds); 174 bounds.normalize(); 175 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds); 176 } 177 DataSource src = new DataSource(bounds, origin); 178 ds.dataSources.add(src); 179 } else { 180 throwException(tr( 181 "Missing mandatory attributes on element ''bounds''. Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.", 182 minlon, minlat, maxlon, maxlat, origin 183 )); 184 } 185 jumpToEnd(); 186 } 187 188 protected Node parseNode() throws XMLStreamException { 189 NodeData nd = new NodeData(); 190 String lat = parser.getAttributeValue(null, "lat"); 191 String lon = parser.getAttributeValue(null, "lon"); 192 if (lat != null && lon != null) { 193 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon))); 194 } 195 readCommon(nd); 196 Node n = new Node(nd.getId(), nd.getVersion()); 197 n.setVisible(nd.isVisible()); 198 n.load(nd); 199 externalIdMap.put(nd.getPrimitiveId(), n); 200 while (true) { 201 int event = parser.next(); 202 if (event == XMLStreamConstants.START_ELEMENT) { 203 if (parser.getLocalName().equals("tag")) { 204 parseTag(n); 205 } else { 206 parseUnknown(); 207 } 208 } else if (event == XMLStreamConstants.END_ELEMENT) 209 return n; 210 } 211 } 212 213 protected Way parseWay() throws XMLStreamException { 214 WayData wd = new WayData(); 215 readCommon(wd); 216 Way w = new Way(wd.getId(), wd.getVersion()); 217 w.setVisible(wd.isVisible()); 218 w.load(wd); 219 externalIdMap.put(wd.getPrimitiveId(), w); 220 221 Collection<Long> nodeIds = new ArrayList<Long>(); 222 while (true) { 223 int event = parser.next(); 224 if (event == XMLStreamConstants.START_ELEMENT) { 225 if (parser.getLocalName().equals("nd")) { 226 nodeIds.add(parseWayNode(w)); 227 } else if (parser.getLocalName().equals("tag")) { 228 parseTag(w); 229 } else { 230 parseUnknown(); 231 } 232 } else if (event == XMLStreamConstants.END_ELEMENT) { 233 break; 234 } 235 } 236 if (w.isDeleted() && !nodeIds.isEmpty()) { 237 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId())); 238 nodeIds = new ArrayList<Long>(); 239 } 240 ways.put(wd.getUniqueId(), nodeIds); 241 return w; 242 } 243 244 private long parseWayNode(Way w) throws XMLStreamException { 245 if (parser.getAttributeValue(null, "ref") == null) { 246 throwException( 247 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId()) 248 ); 249 } 250 long id = getLong("ref"); 251 if (id == 0) { 252 throwException( 253 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id) 254 ); 255 } 256 jumpToEnd(); 257 return id; 258 } 259 260 protected Relation parseRelation() throws XMLStreamException { 261 RelationData rd = new RelationData(); 262 readCommon(rd); 263 Relation r = new Relation(rd.getId(), rd.getVersion()); 264 r.setVisible(rd.isVisible()); 265 r.load(rd); 266 externalIdMap.put(rd.getPrimitiveId(), r); 267 268 Collection<RelationMemberData> members = new ArrayList<RelationMemberData>(); 269 while (true) { 270 int event = parser.next(); 271 if (event == XMLStreamConstants.START_ELEMENT) { 272 if (parser.getLocalName().equals("member")) { 273 members.add(parseRelationMember(r)); 274 } else if (parser.getLocalName().equals("tag")) { 275 parseTag(r); 276 } else { 277 parseUnknown(); 278 } 279 } else if (event == XMLStreamConstants.END_ELEMENT) { 280 break; 281 } 282 } 283 if (r.isDeleted() && !members.isEmpty()) { 284 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId())); 285 members = new ArrayList<RelationMemberData>(); 286 } 287 relations.put(rd.getUniqueId(), members); 288 return r; 289 } 290 291 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { 292 String role = null; 293 OsmPrimitiveType type = null; 294 long id = 0; 295 String value = parser.getAttributeValue(null, "ref"); 296 if (value == null) { 297 throwException(tr("Missing attribute ''ref'' on member in relation {0}.",r.getUniqueId())); 298 } 299 try { 300 id = Long.parseLong(value); 301 } catch(NumberFormatException e) { 302 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),value)); 303 } 304 value = parser.getAttributeValue(null, "type"); 305 if (value == null) { 306 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId()))); 307 } 308 try { 309 type = OsmPrimitiveType.fromApiTypeName(value); 310 } catch(IllegalArgumentException e) { 311 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", Long.toString(id), Long.toString(r.getUniqueId()), value)); 312 } 313 value = parser.getAttributeValue(null, "role"); 314 role = value; 315 316 if (id == 0) { 317 throwException(tr("Incomplete <member> specification with ref=0")); 318 } 319 jumpToEnd(); 320 return new RelationMemberData(role, type, id); 321 } 322 323 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 324 325 Long id = null; 326 if (parser.getAttributeValue(null, "id") != null) { 327 id = getLong("id"); 328 } 329 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value 330 if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) { 331 uploadChangeset = new Changeset(id != null ? id.intValue() : 0); 332 while (true) { 333 int event = parser.next(); 334 if (event == XMLStreamConstants.START_ELEMENT) { 335 if (parser.getLocalName().equals("tag")) { 336 parseTag(uploadChangeset); 337 } else { 338 parseUnknown(); 339 } 340 } else if (event == XMLStreamConstants.END_ELEMENT) 341 return; 342 } 343 } else { 344 jumpToEnd(false); 345 } 346 } 347 348 private void parseTag(Tagged t) throws XMLStreamException { 349 String key = parser.getAttributeValue(null, "k"); 350 String value = parser.getAttributeValue(null, "v"); 351 if (key == null || value == null) { 352 throwException(tr("Missing key or value attribute in tag.")); 353 } 354 t.put(key.intern(), value.intern()); 355 jumpToEnd(); 356 } 357 358 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 359 if (printWarning) { 360 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName())); 361 } 362 while (true) { 363 int event = parser.next(); 364 if (event == XMLStreamConstants.START_ELEMENT) { 365 parseUnknown(false); /* no more warning for inner elements */ 366 } else if (event == XMLStreamConstants.END_ELEMENT) 367 return; 368 } 369 } 370 371 protected void parseUnknown() throws XMLStreamException { 372 parseUnknown(true); 373 } 374 375 /** 376 * When cursor is at the start of an element, moves it to the end tag of that element. 377 * Nested content is skipped. 378 * 379 * This is basically the same code as parseUnknown(), except for the warnings, which 380 * are displayed for inner elements and not at top level. 381 */ 382 private void jumpToEnd(boolean printWarning) throws XMLStreamException { 383 while (true) { 384 int event = parser.next(); 385 if (event == XMLStreamConstants.START_ELEMENT) { 386 parseUnknown(printWarning); 387 } else if (event == XMLStreamConstants.END_ELEMENT) 388 return; 389 } 390 } 391 392 private void jumpToEnd() throws XMLStreamException { 393 jumpToEnd(true); 394 } 395 396 private User createUser(String uid, String name) throws XMLStreamException { 397 if (uid == null) { 398 if (name == null) 399 return null; 400 return User.createLocalUser(name); 401 } 402 try { 403 long id = Long.parseLong(uid); 404 return User.createOsmUser(id, name); 405 } catch(NumberFormatException e) { 406 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid)); 407 } 408 return null; 409 } 410 411 /** 412 * Read out the common attributes and put them into current OsmPrimitive. 413 */ 414 private void readCommon(PrimitiveData current) throws XMLStreamException { 415 current.setId(getLong("id")); 416 if (current.getUniqueId() == 0) { 417 throwException(tr("Illegal object with ID=0.")); 418 } 419 420 String time = parser.getAttributeValue(null, "timestamp"); 421 if (time != null && time.length() != 0) { 422 current.setTimestamp(DateUtils.fromString(time)); 423 } 424 425 // user attribute added in 0.4 API 426 String user = parser.getAttributeValue(null, "user"); 427 // uid attribute added in 0.6 API 428 String uid = parser.getAttributeValue(null, "uid"); 429 current.setUser(createUser(uid, user)); 430 431 // visible attribute added in 0.4 API 432 String visible = parser.getAttributeValue(null, "visible"); 433 if (visible != null) { 434 current.setVisible(Boolean.parseBoolean(visible)); 435 } 436 437 String versionString = parser.getAttributeValue(null, "version"); 438 int version = 0; 439 if (versionString != null) { 440 try { 441 version = Integer.parseInt(versionString); 442 } catch(NumberFormatException e) { 443 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString)); 444 } 445 if (ds.getVersion().equals("0.6")){ 446 if (version <= 0 && current.getUniqueId() > 0) { 447 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", Long.toString(current.getUniqueId()), versionString)); 448 } else if (version < 0 && current.getUniqueId() <= 0) { 449 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.6")); 450 version = 0; 451 } 452 } else if (ds.getVersion().equals("0.5")) { 453 if (version <= 0 && current.getUniqueId() > 0) { 454 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5")); 455 version = 1; 456 } else if (version < 0 && current.getUniqueId() <= 0) { 457 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 0, "0.5")); 458 version = 0; 459 } 460 } else { 461 // should not happen. API version has been checked before 462 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); 463 } 464 } else { 465 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 466 // 467 if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.6")) { 468 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); 469 } else if (current.getUniqueId() > 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) { 470 // default version in 0.5 files for existing primitives 471 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", current.getUniqueId(), version, 1, "0.5")); 472 version= 1; 473 } else if (current.getUniqueId() <= 0 && ds.getVersion() != null && ds.getVersion().equals("0.5")) { 474 // default version in 0.5 files for new primitives, no warning necessary. This is 475 // (was) legal in API 0.5 476 version= 0; 477 } 478 } 479 current.setVersion(version); 480 481 String action = parser.getAttributeValue(null, "action"); 482 if (action == null) { 483 // do nothing 484 } else if (action.equals("delete")) { 485 current.setDeleted(true); 486 current.setModified(current.isVisible()); 487 } else if (action.equals("modify")) { 488 current.setModified(true); 489 } 490 491 String v = parser.getAttributeValue(null, "changeset"); 492 if (v == null) { 493 current.setChangesetId(0); 494 } else { 495 try { 496 current.setChangesetId(Integer.parseInt(v)); 497 } catch(NumberFormatException e) { 498 if (current.getUniqueId() <= 0) { 499 // for a new primitive we just log a warning 500 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId())); 501 current.setChangesetId(0); 502 } else { 503 // for an existing primitive this is a problem 504 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 505 } 506 } 507 if (current.getChangesetId() <=0) { 508 if (current.getUniqueId() <= 0) { 509 // for a new primitive we just log a warning 510 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", v, current.getUniqueId())); 511 current.setChangesetId(0); 512 } else { 513 // for an existing primitive this is a problem 514 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 515 } 516 } 517 } 518 } 519 520 private long getLong(String name) throws XMLStreamException { 521 String value = parser.getAttributeValue(null, name); 522 if (value == null) { 523 throwException(tr("Missing required attribute ''{0}''.",name)); 524 } 525 try { 526 return Long.parseLong(value); 527 } catch(NumberFormatException e) { 528 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.",name, value)); 529 } 530 return 0; // should not happen 531 } 532 533 private static class OsmParsingException extends XMLStreamException { 534 public OsmParsingException() { 535 super(); 536 } 537 538 public OsmParsingException(String msg) { 539 super(msg); 540 } 541 542 public OsmParsingException(String msg, Location location) { 543 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */ 544 this.location = location; 545 } 546 547 public OsmParsingException(String msg, Location location, Throwable th) { 548 super(msg, th); 549 this.location = location; 550 } 551 552 public OsmParsingException(String msg, Throwable th) { 553 super(msg, th); 554 } 555 556 public OsmParsingException(Throwable th) { 557 super(th); 558 } 559 560 @Override 561 public String getMessage() { 562 String msg = super.getMessage(); 563 if (msg == null) { 564 msg = getClass().getName(); 565 } 566 if (getLocation() == null) 567 return msg; 568 msg = msg + " " + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber()); 569 return msg; 570 } 571 } 572 573 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 574 if (progressMonitor == null) { 575 progressMonitor = NullProgressMonitor.INSTANCE; 576 } 577 ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() { 578 @Override public void operationCanceled() { 579 cancel = true; 580 } 581 }; 582 progressMonitor.addCancelListener(cancelListener); 583 CheckParameterUtil.ensureParameterNotNull(source, "source"); 584 try { 585 progressMonitor.beginTask(tr("Prepare OSM data...", 2)); 586 progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); 587 588 InputStreamReader ir = UTFInputStreamReader.create(source, "UTF-8"); 589 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir); 590 setParser(parser); 591 parse(); 592 progressMonitor.worked(1); 593 594 progressMonitor.indeterminateSubTask(tr("Preparing data set...")); 595 prepareDataSet(); 596 progressMonitor.worked(1); 597 598 // iterate over registered postprocessors and give them each a chance 599 // to modify the dataset we have just loaded. 600 if (postprocessors != null) { 601 for (OsmServerReadPostprocessor pp : postprocessors) { 602 pp.postprocessDataSet(getDataSet(), progressMonitor); 603 } 604 } 605 return getDataSet(); 606 } catch(IllegalDataException e) { 607 throw e; 608 } catch(OsmParsingException e) { 609 throw new IllegalDataException(e.getMessage(), e); 610 } catch(XMLStreamException e) { 611 String msg = e.getMessage(); 612 Pattern p = Pattern.compile("Message: (.+)"); 613 Matcher m = p.matcher(msg); 614 if (m.find()) { 615 msg = m.group(1); 616 } 617 if (e.getLocation() != null) 618 throw new IllegalDataException(tr("Line {0} column {1}: ", e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 619 else 620 throw new IllegalDataException(msg, e); 621 } catch(Exception e) { 622 throw new IllegalDataException(e); 623 } finally { 624 progressMonitor.finishTask(); 625 progressMonitor.removeCancelListener(cancelListener); 626 } 627 } 628 629 /** 630 * Parse the given input source and return the dataset. 631 * 632 * @param source the source input stream. Must not be null. 633 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 634 * 635 * @return the dataset with the parsed data 636 * @throws IllegalDataException thrown if the an error was found while parsing the data from the source 637 * @throws IllegalArgumentException thrown if source is null 638 */ 639 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 640 return new OsmReader().doParseDataSet(source, progressMonitor); 641 } 642}