001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.awt.GridBagConstraints; 008import java.awt.event.ActionEvent; 009import java.awt.event.ActionListener; 010import java.io.BufferedReader; 011import java.io.IOException; 012import java.io.InputStream; 013import java.text.MessageFormat; 014import java.util.ArrayList; 015import java.util.Arrays; 016import java.util.Collection; 017import java.util.HashMap; 018import java.util.List; 019import java.util.Locale; 020import java.util.Map; 021import java.util.Map.Entry; 022import java.util.Set; 023import java.util.regex.Matcher; 024import java.util.regex.Pattern; 025import java.util.regex.PatternSyntaxException; 026 027import javax.swing.JCheckBox; 028import javax.swing.JLabel; 029import javax.swing.JPanel; 030 031import org.openstreetmap.josm.Main; 032import org.openstreetmap.josm.command.ChangePropertyCommand; 033import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 034import org.openstreetmap.josm.command.Command; 035import org.openstreetmap.josm.command.SequenceCommand; 036import org.openstreetmap.josm.data.osm.OsmPrimitive; 037import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 038import org.openstreetmap.josm.data.osm.OsmUtils; 039import org.openstreetmap.josm.data.osm.Tag; 040import org.openstreetmap.josm.data.validation.FixableTestError; 041import org.openstreetmap.josm.data.validation.Severity; 042import org.openstreetmap.josm.data.validation.Test.TagTest; 043import org.openstreetmap.josm.data.validation.TestError; 044import org.openstreetmap.josm.data.validation.util.Entities; 045import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference; 046import org.openstreetmap.josm.gui.progress.ProgressMonitor; 047import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 048import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 049import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 050import org.openstreetmap.josm.gui.tagging.presets.items.Check; 051import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 052import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 053import org.openstreetmap.josm.gui.widgets.EditableList; 054import org.openstreetmap.josm.io.CachedFile; 055import org.openstreetmap.josm.io.UTFInputStreamReader; 056import org.openstreetmap.josm.tools.GBC; 057import org.openstreetmap.josm.tools.MultiMap; 058import org.openstreetmap.josm.tools.Utils; 059 060/** 061 * Check for misspelled or wrong tags 062 * 063 * @author frsantos 064 * @since 3669 065 */ 066public class TagChecker extends TagTest { 067 068 /** The config file of ignored tags */ 069 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 070 /** The config file of dictionary words */ 071 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 072 073 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 074 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 075 /** The spell check preset values */ 076 private static volatile MultiMap<String, String> presetsValueData; 077 /** The TagChecker data */ 078 private static final List<CheckerData> checkerData = new ArrayList<>(); 079 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 080 private static final List<String> ignoreDataEquals = new ArrayList<>(); 081 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 082 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 083 084 /** The preferences prefix */ 085 protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName(); 086 087 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 088 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 089 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 090 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 091 092 public static final String PREF_SOURCES = PREFIX + ".source"; 093 094 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload"; 095 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload"; 096 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload"; 097 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload"; 098 099 protected boolean checkKeys; 100 protected boolean checkValues; 101 protected boolean checkComplex; 102 protected boolean checkFixmes; 103 104 protected JCheckBox prefCheckKeys; 105 protected JCheckBox prefCheckValues; 106 protected JCheckBox prefCheckComplex; 107 protected JCheckBox prefCheckFixmes; 108 protected JCheckBox prefCheckPaint; 109 110 protected JCheckBox prefCheckKeysBeforeUpload; 111 protected JCheckBox prefCheckValuesBeforeUpload; 112 protected JCheckBox prefCheckComplexBeforeUpload; 113 protected JCheckBox prefCheckFixmesBeforeUpload; 114 protected JCheckBox prefCheckPaintBeforeUpload; 115 116 protected static final int EMPTY_VALUES = 1200; 117 protected static final int INVALID_KEY = 1201; 118 protected static final int INVALID_VALUE = 1202; 119 protected static final int FIXME = 1203; 120 protected static final int INVALID_SPACE = 1204; 121 protected static final int INVALID_KEY_SPACE = 1205; 122 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 123 protected static final int LONG_VALUE = 1208; 124 protected static final int LONG_KEY = 1209; 125 protected static final int LOW_CHAR_VALUE = 1210; 126 protected static final int LOW_CHAR_KEY = 1211; 127 protected static final int MISSPELLED_VALUE = 1212; 128 protected static final int MISSPELLED_KEY = 1213; 129 /** 1250 and up is used by tagcheck */ 130 131 protected EditableList sourcesList; 132 133 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE); 134 135 /** 136 * Constructor 137 */ 138 public TagChecker() { 139 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 140 } 141 142 @Override 143 public void initialize() throws IOException { 144 initializeData(); 145 initializePresets(); 146 } 147 148 /** 149 * Reads the spellcheck file into a HashMap. 150 * The data file is a list of words, beginning with +/-. If it starts with +, 151 * the word is valid, but if it starts with -, the word should be replaced 152 * by the nearest + word before this. 153 * 154 * @throws IOException if any I/O error occurs 155 */ 156 private static void initializeData() throws IOException { 157 checkerData.clear(); 158 ignoreDataStartsWith.clear(); 159 ignoreDataEquals.clear(); 160 ignoreDataEndsWith.clear(); 161 ignoreDataTag.clear(); 162 harmonizedKeys.clear(); 163 164 StringBuilder errorSources = new StringBuilder(); 165 for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) { 166 try ( 167 InputStream s = new CachedFile(source).getInputStream(); 168 BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s)); 169 ) { 170 String okValue = null; 171 boolean tagcheckerfile = false; 172 boolean ignorefile = false; 173 boolean isFirstLine = true; 174 String line; 175 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) { 176 if (line.startsWith("#")) { 177 if (line.startsWith("# JOSM TagChecker")) { 178 tagcheckerfile = true; 179 if (!DEFAULT_SOURCES.contains(source)) { 180 Main.info(tr("Adding {0} to tag checker", source)); 181 } 182 } else 183 if (line.startsWith("# JOSM IgnoreTags")) { 184 ignorefile = true; 185 if (!DEFAULT_SOURCES.contains(source)) { 186 Main.info(tr("Adding {0} to ignore tags", source)); 187 } 188 } 189 } else if (ignorefile) { 190 line = line.trim(); 191 if (line.length() < 4) { 192 continue; 193 } 194 195 String key = line.substring(0, 2); 196 line = line.substring(2); 197 198 switch (key) { 199 case "S:": 200 ignoreDataStartsWith.add(line); 201 break; 202 case "E:": 203 ignoreDataEquals.add(line); 204 break; 205 case "F:": 206 ignoreDataEndsWith.add(line); 207 break; 208 case "K:": 209 ignoreDataTag.add(Tag.ofString(line)); 210 } 211 } else if (tagcheckerfile) { 212 if (!line.isEmpty()) { 213 CheckerData d = new CheckerData(); 214 String err = d.getData(line); 215 216 if (err == null) { 217 checkerData.add(d); 218 } else { 219 Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line)); 220 } 221 } 222 } else if (line.charAt(0) == '+') { 223 okValue = line.substring(1); 224 } else if (line.charAt(0) == '-' && okValue != null) { 225 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue); 226 } else { 227 Main.error(tr("Invalid spellcheck line: {0}", line)); 228 } 229 if (isFirstLine) { 230 isFirstLine = false; 231 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 232 Main.info(tr("Adding {0} to spellchecker", source)); 233 } 234 } 235 } 236 } catch (IOException e) { 237 errorSources.append(source).append('\n'); 238 } 239 } 240 241 if (errorSources.length() > 0) 242 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources)); 243 } 244 245 /** 246 * Reads the presets data. 247 * 248 */ 249 public static void initializePresets() { 250 251 if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true)) 252 return; 253 254 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 255 if (!presets.isEmpty()) { 256 presetsValueData = new MultiMap<>(); 257 for (String a : OsmPrimitive.getUninterestingKeys()) { 258 presetsValueData.putVoid(a); 259 } 260 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead) 261 for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys", 262 Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) { 263 presetsValueData.putVoid(a); 264 } 265 for (TaggingPreset p : presets) { 266 for (TaggingPresetItem i : p.data) { 267 if (i instanceof KeyedItem) { 268 addPresetValue(p, (KeyedItem) i); 269 } else if (i instanceof CheckGroup) { 270 for (Check c : ((CheckGroup) i).checks) { 271 addPresetValue(p, c); 272 } 273 } 274 } 275 } 276 } 277 } 278 279 private static void addPresetValue(TaggingPreset p, KeyedItem ky) { 280 Collection<String> values = ky.getValues(); 281 if (ky.key != null && values != null) { 282 try { 283 presetsValueData.putAll(ky.key, values); 284 harmonizedKeys.put(harmonizeKey(ky.key), ky.key); 285 } catch (NullPointerException e) { 286 Main.error(p+": Unable to initialize "+ky); 287 } 288 } 289 } 290 291 /** 292 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters) 293 * @param s string to check 294 * @return {@code true} if {@code s} contains characters with code below 0x20 295 */ 296 private static boolean containsLow(String s) { 297 if (s == null) 298 return false; 299 for (int i = 0; i < s.length(); i++) { 300 if (s.charAt(i) < 0x20) 301 return true; 302 } 303 return false; 304 } 305 306 /** 307 * Determines if the given key is in internal presets. 308 * @param key key 309 * @return {@code true} if the given key is in internal presets 310 * @since 9023 311 */ 312 public static boolean isKeyInPresets(String key) { 313 return presetsValueData.get(key) != null; 314 } 315 316 /** 317 * Determines if the given tag is in internal presets. 318 * @param key key 319 * @param value value 320 * @return {@code true} if the given tag is in internal presets 321 * @since 9023 322 */ 323 public static boolean isTagInPresets(String key, String value) { 324 final Set<String> values = presetsValueData.get(key); 325 return values != null && (values.isEmpty() || values.contains(value)); 326 } 327 328 /** 329 * Returns the list of ignored tags. 330 * @return the list of ignored tags 331 * @since 9023 332 */ 333 public static List<Tag> getIgnoredTags() { 334 return new ArrayList<>(ignoreDataTag); 335 } 336 337 /** 338 * Determines if the given tag is ignored for checks "key/tag not in presets". 339 * @param key key 340 * @param value value 341 * @return {@code true} if the given tag is ignored 342 * @since 9023 343 */ 344 public static boolean isTagIgnored(String key, String value) { 345 boolean tagInPresets = isTagInPresets(key, value); 346 boolean ignore = false; 347 348 for (String a : ignoreDataStartsWith) { 349 if (key.startsWith(a)) { 350 ignore = true; 351 } 352 } 353 for (String a : ignoreDataEquals) { 354 if (key.equals(a)) { 355 ignore = true; 356 } 357 } 358 for (String a : ignoreDataEndsWith) { 359 if (key.endsWith(a)) { 360 ignore = true; 361 } 362 } 363 364 if (!tagInPresets) { 365 for (Tag a : ignoreDataTag) { 366 if (key.equals(a.getKey()) && value.equals(a.getValue())) { 367 ignore = true; 368 } 369 } 370 } 371 return ignore; 372 } 373 374 /** 375 * Checks the primitive tags 376 * @param p The primitive to check 377 */ 378 @Override 379 public void check(OsmPrimitive p) { 380 // Just a collection to know if a primitive has been already marked with error 381 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 382 383 if (checkComplex) { 384 Map<String, String> keys = p.getKeys(); 385 for (CheckerData d : checkerData) { 386 if (d.match(p, keys)) { 387 errors.add(new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"), 388 d.getDescription(), d.getDescriptionOrig(), d.getCode(), p)); 389 withErrors.put(p, "TC"); 390 } 391 } 392 } 393 394 for (Entry<String, String> prop : p.getKeys().entrySet()) { 395 String s = marktr("Key ''{0}'' invalid."); 396 String key = prop.getKey(); 397 String value = prop.getValue(); 398 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) { 399 errors.add(new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"), 400 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p)); 401 withErrors.put(p, "ICV"); 402 } 403 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) { 404 errors.add(new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"), 405 tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p)); 406 withErrors.put(p, "ICK"); 407 } 408 if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) { 409 errors.add(new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"), 410 tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p)); 411 withErrors.put(p, "LV"); 412 } 413 if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) { 414 errors.add(new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"), 415 tr(s, key), MessageFormat.format(s, key), LONG_KEY, p)); 416 withErrors.put(p, "LK"); 417 } 418 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) { 419 errors.add(new TestError(this, Severity.WARNING, tr("Tags with empty values"), 420 tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p)); 421 withErrors.put(p, "EV"); 422 } 423 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 424 errors.add(new TestError(this, Severity.WARNING, tr("Invalid white space in property key"), 425 tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p)); 426 withErrors.put(p, "IPK"); 427 } 428 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) { 429 errors.add(new TestError(this, Severity.WARNING, tr("Property values start or end with white space"), 430 tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p)); 431 withErrors.put(p, "SPACE"); 432 } 433 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 434 errors.add(new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"), 435 tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p)); 436 withErrors.put(p, "HTML"); 437 } 438 if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null) { 439 if (!isTagIgnored(key, value)) { 440 if (!isKeyInPresets(key)) { 441 String prettifiedKey = harmonizeKey(key); 442 String fixedKey = harmonizedKeys.get(prettifiedKey); 443 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 444 // misspelled preset key 445 String i = marktr("Key ''{0}'' looks like ''{1}''."); 446 errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property key"), 447 tr(i, key, fixedKey), 448 MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p, 449 new ChangePropertyKeyCommand(p, key, fixedKey))); 450 withErrors.put(p, "WPK"); 451 } else { 452 String i = marktr("Key ''{0}'' not in presets."); 453 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property key"), 454 tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p)); 455 withErrors.put(p, "UPK"); 456 } 457 } else if (!isTagInPresets(key, value)) { 458 // try to fix common typos and check again if value is still unknown 459 String fixedValue = harmonizeValue(prop.getValue()); 460 Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key)); 461 if (possibleValues.containsKey(fixedValue)) { 462 fixedValue = possibleValues.get(fixedValue); 463 // misspelled preset value 464 String i = marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."); 465 errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property value"), 466 tr(i, prop.getValue(), key, fixedValue), MessageFormat.format(i, prop.getValue(), fixedValue), 467 MISSPELLED_VALUE, p, new ChangePropertyCommand(p, key, fixedValue))); 468 withErrors.put(p, "WPV"); 469 } else { 470 // unknown preset value 471 String i = marktr("Value ''{0}'' for key ''{1}'' not in presets."); 472 errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property value"), 473 tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p)); 474 withErrors.put(p, "UPV"); 475 } 476 } 477 } 478 } 479 if (checkFixmes && key != null && value != null && !value.isEmpty()) { 480 if ((value.toLowerCase(Locale.ENGLISH).contains("fixme") 481 || value.contains("check and delete") 482 || key.contains("todo") || key.toLowerCase(Locale.ENGLISH).contains("fixme")) 483 && !withErrors.contains(p, "FIXME")) { 484 errors.add(new TestError(this, Severity.OTHER, 485 tr("FIXMES"), FIXME, p)); 486 withErrors.put(p, "FIXME"); 487 } 488 } 489 } 490 } 491 492 private static Map<String, String> getPossibleValues(Set<String> values) { 493 // generate a map with common typos 494 Map<String, String> map = new HashMap<>(); 495 if (values != null) { 496 for (String value : values) { 497 map.put(value, value); 498 if (value.contains("_")) { 499 map.put(value.replace("_", ""), value); 500 } 501 } 502 } 503 return map; 504 } 505 506 private static String harmonizeKey(String key) { 507 key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'); 508 return Utils.strip(key, "-_;:,"); 509 } 510 511 private static String harmonizeValue(String value) { 512 value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'); 513 return Utils.strip(value, "-_;:,"); 514 } 515 516 @Override 517 public void startTest(ProgressMonitor monitor) { 518 super.startTest(monitor); 519 checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true); 520 if (isBeforeUpload) { 521 checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 522 } 523 524 checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true); 525 if (isBeforeUpload) { 526 checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 527 } 528 529 checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true); 530 if (isBeforeUpload) { 531 checkComplex = checkValues && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 532 } 533 534 checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true); 535 if (isBeforeUpload) { 536 checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 537 } 538 } 539 540 @Override 541 public void visit(Collection<OsmPrimitive> selection) { 542 if (checkKeys || checkValues || checkComplex || checkFixmes) { 543 super.visit(selection); 544 } 545 } 546 547 @Override 548 public void addGui(JPanel testPanel) { 549 GBC a = GBC.eol(); 550 a.anchor = GridBagConstraints.EAST; 551 552 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 553 554 prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true)); 555 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 556 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 557 558 prefCheckKeysBeforeUpload = new JCheckBox(); 559 prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 560 testPanel.add(prefCheckKeysBeforeUpload, a); 561 562 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true)); 563 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 564 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 565 566 prefCheckComplexBeforeUpload = new JCheckBox(); 567 prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 568 testPanel.add(prefCheckComplexBeforeUpload, a); 569 570 final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES); 571 sourcesList = new EditableList(tr("TagChecker source")); 572 sourcesList.setItems(sources); 573 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 574 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 575 576 ActionListener disableCheckActionListener = new ActionListener() { 577 @Override 578 public void actionPerformed(ActionEvent e) { 579 handlePrefEnable(); 580 } 581 }; 582 prefCheckKeys.addActionListener(disableCheckActionListener); 583 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 584 prefCheckComplex.addActionListener(disableCheckActionListener); 585 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 586 587 handlePrefEnable(); 588 589 prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true)); 590 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 591 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 592 593 prefCheckValuesBeforeUpload = new JCheckBox(); 594 prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 595 testPanel.add(prefCheckValuesBeforeUpload, a); 596 597 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true)); 598 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value.")); 599 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 600 601 prefCheckFixmesBeforeUpload = new JCheckBox(); 602 prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 603 testPanel.add(prefCheckFixmesBeforeUpload, a); 604 } 605 606 public void handlePrefEnable() { 607 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 608 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 609 sourcesList.setEnabled(selected); 610 } 611 612 @Override 613 public boolean ok() { 614 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 615 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 616 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 617 618 Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 619 Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 620 Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 621 Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 622 Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 623 Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 624 Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 625 Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 626 return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems()); 627 } 628 629 @Override 630 public Command fixError(TestError testError) { 631 List<Command> commands = new ArrayList<>(50); 632 633 if (testError instanceof FixableTestError) { 634 commands.add(testError.getFix()); 635 } else { 636 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 637 for (OsmPrimitive p : primitives) { 638 Map<String, String> tags = p.getKeys(); 639 if (tags == null || tags.isEmpty()) { 640 continue; 641 } 642 643 for (Entry<String, String> prop: tags.entrySet()) { 644 String key = prop.getKey(); 645 String value = prop.getValue(); 646 if (value == null || value.trim().isEmpty()) { 647 commands.add(new ChangePropertyCommand(p, key, null)); 648 } else if (value.startsWith(" ") || value.endsWith(" ")) { 649 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value))); 650 } else if (key.startsWith(" ") || key.endsWith(" ")) { 651 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key))); 652 } else { 653 String evalue = Entities.unescape(value); 654 if (!evalue.equals(value)) { 655 commands.add(new ChangePropertyCommand(p, key, evalue)); 656 } 657 } 658 } 659 } 660 } 661 662 if (commands.isEmpty()) 663 return null; 664 if (commands.size() == 1) 665 return commands.get(0); 666 667 return new SequenceCommand(tr("Fix tags"), commands); 668 } 669 670 @Override 671 public boolean isFixable(TestError testError) { 672 if (testError.getTester() instanceof TagChecker) { 673 int code = testError.getCode(); 674 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || 675 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE; 676 } 677 678 return false; 679 } 680 681 protected static class CheckerData { 682 private String description; 683 protected List<CheckerElement> data = new ArrayList<>(); 684 private OsmPrimitiveType type; 685 private int code; 686 protected Severity severity; 687 protected static final int TAG_CHECK_ERROR = 1250; 688 protected static final int TAG_CHECK_WARN = 1260; 689 protected static final int TAG_CHECK_INFO = 1270; 690 691 protected static class CheckerElement { 692 public Object tag; 693 public Object value; 694 public boolean noMatch; 695 public boolean tagAll; 696 public boolean valueAll; 697 public boolean valueBool; 698 699 private static Pattern getPattern(String str) throws PatternSyntaxException { 700 if (str.endsWith("/i")) 701 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE); 702 if (str.endsWith("/")) 703 return Pattern.compile(str.substring(1, str.length()-1)); 704 705 throw new IllegalStateException(); 706 } 707 708 public CheckerElement(String exp) throws PatternSyntaxException { 709 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp); 710 m.matches(); 711 712 String n = m.group(1).trim(); 713 714 if ("*".equals(n)) { 715 tagAll = true; 716 } else { 717 tag = n.startsWith("/") ? getPattern(n) : n; 718 noMatch = "!=".equals(m.group(2)); 719 n = m.group(3).trim(); 720 if ("*".equals(n)) { 721 valueAll = true; 722 } else if ("BOOLEAN_TRUE".equals(n)) { 723 valueBool = true; 724 value = OsmUtils.trueval; 725 } else if ("BOOLEAN_FALSE".equals(n)) { 726 valueBool = true; 727 value = OsmUtils.falseval; 728 } else { 729 value = n.startsWith("/") ? getPattern(n) : n; 730 } 731 } 732 } 733 734 public boolean match(Map<String, String> keys) { 735 for (Entry<String, String> prop: keys.entrySet()) { 736 String key = prop.getKey(); 737 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue(); 738 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag))) 739 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value)))) 740 return !noMatch; 741 } 742 return noMatch; 743 } 744 } 745 746 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$"); 747 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *"); 748 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *"); 749 750 public String getData(final String str) { 751 Matcher m = CLEAN_STR_PATTERN.matcher(str); 752 String trimmed = m.replaceFirst("").trim(); 753 try { 754 description = m.group(1); 755 if (description != null && description.isEmpty()) { 756 description = null; 757 } 758 } catch (IllegalStateException e) { 759 description = null; 760 } 761 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3); 762 switch (n[0]) { 763 case "way": 764 type = OsmPrimitiveType.WAY; 765 break; 766 case "node": 767 type = OsmPrimitiveType.NODE; 768 break; 769 case "relation": 770 type = OsmPrimitiveType.RELATION; 771 break; 772 case "*": 773 type = null; 774 break; 775 default: 776 return tr("Could not find element type"); 777 } 778 if (n.length != 3) 779 return tr("Incorrect number of parameters"); 780 781 switch (n[1]) { 782 case "W": 783 severity = Severity.WARNING; 784 code = TAG_CHECK_WARN; 785 break; 786 case "E": 787 severity = Severity.ERROR; 788 code = TAG_CHECK_ERROR; 789 break; 790 case "I": 791 severity = Severity.OTHER; 792 code = TAG_CHECK_INFO; 793 break; 794 default: 795 return tr("Could not find warning level"); 796 } 797 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) { 798 try { 799 data.add(new CheckerElement(exp)); 800 } catch (IllegalStateException e) { 801 return tr("Illegal expression ''{0}''", exp); 802 } catch (PatternSyntaxException e) { 803 return tr("Illegal regular expression ''{0}''", exp); 804 } 805 } 806 return null; 807 } 808 809 public boolean match(OsmPrimitive osm, Map<String, String> keys) { 810 if (type != null && OsmPrimitiveType.from(osm) != type) 811 return false; 812 813 for (CheckerElement ce : data) { 814 if (!ce.match(keys)) 815 return false; 816 } 817 return true; 818 } 819 820 public String getDescription() { 821 return tr(description); 822 } 823 824 public String getDescriptionOrig() { 825 return description; 826 } 827 828 public Severity getSeverity() { 829 return severity; 830 } 831 832 public int getCode() { 833 if (type == null) 834 return code; 835 836 return code + type.ordinal() + 1; 837 } 838 } 839}