001// License: GPL. See LICENSE file for details.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.Dimension;
008import java.awt.GridBagConstraints;
009import java.awt.GridBagLayout;
010import java.awt.event.ActionEvent;
011import java.awt.event.ActionListener;
012import java.io.BufferedReader;
013import java.io.FileNotFoundException;
014import java.io.IOException;
015import java.io.InputStreamReader;
016import java.io.UnsupportedEncodingException;
017import java.text.MessageFormat;
018import java.util.ArrayList;
019import java.util.Arrays;
020import java.util.Collection;
021import java.util.HashMap;
022import java.util.List;
023import java.util.Map;
024import java.util.Map.Entry;
025import java.util.Set;
026import java.util.regex.Matcher;
027import java.util.regex.Pattern;
028import java.util.regex.PatternSyntaxException;
029
030import javax.swing.DefaultListModel;
031import javax.swing.JButton;
032import javax.swing.JCheckBox;
033import javax.swing.JLabel;
034import javax.swing.JList;
035import javax.swing.JOptionPane;
036import javax.swing.JPanel;
037import javax.swing.JScrollPane;
038
039import org.openstreetmap.josm.Main;
040import org.openstreetmap.josm.command.ChangePropertyCommand;
041import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
042import org.openstreetmap.josm.command.Command;
043import org.openstreetmap.josm.command.SequenceCommand;
044import org.openstreetmap.josm.data.osm.Node;
045import org.openstreetmap.josm.data.osm.OsmPrimitive;
046import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
047import org.openstreetmap.josm.data.osm.OsmUtils;
048import org.openstreetmap.josm.data.osm.Relation;
049import org.openstreetmap.josm.data.osm.Way;
050import org.openstreetmap.josm.data.validation.Severity;
051import org.openstreetmap.josm.data.validation.Test;
052import org.openstreetmap.josm.data.validation.TestError;
053import org.openstreetmap.josm.data.validation.util.Entities;
054import org.openstreetmap.josm.gui.preferences.ValidatorPreference;
055import org.openstreetmap.josm.gui.preferences.map.TaggingPresetPreference;
056import org.openstreetmap.josm.gui.progress.ProgressMonitor;
057import org.openstreetmap.josm.gui.tagging.TaggingPreset;
058import org.openstreetmap.josm.gui.tagging.TaggingPresetItem;
059import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.Check;
060import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.CheckGroup;
061import org.openstreetmap.josm.gui.tagging.TaggingPresetItems.KeyedItem;
062import org.openstreetmap.josm.io.MirroredInputStream;
063import org.openstreetmap.josm.tools.GBC;
064import org.openstreetmap.josm.tools.MultiMap;
065import org.openstreetmap.josm.tools.Utils;
066
067/**
068 * Check for misspelled or wrong tags
069 *
070 * @author frsantos
071 */
072public class TagChecker extends Test {
073    
074    /** The default data file of tagchecker rules */
075    public static final String DATA_FILE = "resource://data/validator/tagchecker.cfg";
076    /** The config file of ignored tags */
077    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
078    /** The config file of dictionary words */
079    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
080
081    /** The spell check key substitutions: the key should be substituted by the value */
082    protected static Map<String, String> spellCheckKeyData;
083    /** The spell check preset values */
084    protected static MultiMap<String, String> presetsValueData;
085    /** The TagChecker data */
086    protected static final List<CheckerData> checkerData = new ArrayList<CheckerData>();
087    protected static final List<String> ignoreDataStartsWith = new ArrayList<String>();
088    protected static final List<String> ignoreDataEquals = new ArrayList<String>();
089    protected static final List<String> ignoreDataEndsWith = new ArrayList<String>();
090    protected static final List<IgnoreKeyPair> ignoreDataKeyPair = new ArrayList<IgnoreKeyPair>();
091
092    /** The preferences prefix */
093    protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
094
095    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
096    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
097    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
098    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
099
100    public static final String PREF_SOURCES = PREFIX + ".sources";
101    public static final String PREF_USE_DATA_FILE = PREFIX + ".usedatafile";
102    public static final String PREF_USE_IGNORE_FILE = PREFIX + ".useignorefile";
103    public static final String PREF_USE_SPELL_FILE = PREFIX + ".usespellfile";
104
105    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
106    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
107    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
108    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
109
110    protected boolean checkKeys = false;
111    protected boolean checkValues = false;
112    protected boolean checkComplex = false;
113    protected boolean checkFixmes = false;
114
115    protected JCheckBox prefCheckKeys;
116    protected JCheckBox prefCheckValues;
117    protected JCheckBox prefCheckComplex;
118    protected JCheckBox prefCheckFixmes;
119    protected JCheckBox prefCheckPaint;
120
121    protected JCheckBox prefCheckKeysBeforeUpload;
122    protected JCheckBox prefCheckValuesBeforeUpload;
123    protected JCheckBox prefCheckComplexBeforeUpload;
124    protected JCheckBox prefCheckFixmesBeforeUpload;
125    protected JCheckBox prefCheckPaintBeforeUpload;
126
127    protected JCheckBox prefUseDataFile;
128    protected JCheckBox prefUseIgnoreFile;
129    protected JCheckBox prefUseSpellFile;
130
131    protected JButton addSrcButton;
132    protected JButton editSrcButton;
133    protected JButton deleteSrcButton;
134
135    protected static final int EMPTY_VALUES      = 1200;
136    protected static final int INVALID_KEY       = 1201;
137    protected static final int INVALID_VALUE     = 1202;
138    protected static final int FIXME             = 1203;
139    protected static final int INVALID_SPACE     = 1204;
140    protected static final int INVALID_KEY_SPACE = 1205;
141    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
142    protected static final int LONG_VALUE        = 1208;
143    protected static final int LONG_KEY          = 1209;
144    protected static final int LOW_CHAR_VALUE    = 1210;
145    protected static final int LOW_CHAR_KEY      = 1211;
146    /** 1250 and up is used by tagcheck */
147
148    /** List of sources for spellcheck data */
149    protected JList sourcesList;
150
151    protected static final Entities entities = new Entities();
152
153    /**
154     * Constructor
155     */
156    public TagChecker() {
157        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
158    }
159
160    @Override
161    public void initialize() throws IOException {
162        initializeData();
163        initializePresets();
164    }
165
166    /**
167     * Reads the spellcheck file into a HashMap.
168     * The data file is a list of words, beginning with +/-. If it starts with +,
169     * the word is valid, but if it starts with -, the word should be replaced
170     * by the nearest + word before this.
171     *
172     * @throws FileNotFoundException
173     * @throws IOException
174     */
175    private static void initializeData() throws IOException {
176        checkerData.clear();
177        ignoreDataStartsWith.clear();
178        ignoreDataEquals.clear();
179        ignoreDataEndsWith.clear();
180        ignoreDataKeyPair.clear();
181
182        spellCheckKeyData = new HashMap<String, String>();
183        String sources = Main.pref.get( PREF_SOURCES, "");
184        if (Main.pref.getBoolean(PREF_USE_DATA_FILE, true)) {
185            if (sources == null || sources.length() == 0) {
186                sources = DATA_FILE;
187            } else {
188                sources = DATA_FILE + ";" + sources;
189            }
190        }
191        if (Main.pref.getBoolean(PREF_USE_IGNORE_FILE, true)) {
192            if (sources == null || sources.length() == 0) {
193                sources = IGNORE_FILE;
194            } else {
195                sources = IGNORE_FILE + ";" + sources;
196            }
197        }
198        if (Main.pref.getBoolean(PREF_USE_SPELL_FILE, true)) {
199            if( sources == null || sources.length() == 0) {
200                sources = SPELL_FILE;
201            } else {
202                sources = SPELL_FILE + ";" + sources;
203            }
204        }
205
206        String errorSources = "";
207        if (sources.length() == 0)
208            return;
209        for (String source : sources.split(";")) {
210            BufferedReader reader = null;
211            try {
212                MirroredInputStream s = new MirroredInputStream(source);
213                InputStreamReader r;
214                try {
215                    r = new InputStreamReader(s, "UTF-8");
216                } catch (UnsupportedEncodingException e) {
217                    r = new InputStreamReader(s);
218                }
219                reader = new BufferedReader(r);
220
221                String okValue = null;
222                boolean tagcheckerfile = false;
223                boolean ignorefile = false;
224                String line;
225                while ((line = reader.readLine()) != null && (tagcheckerfile || line.length() != 0)) {
226                    if (line.startsWith("#")) {
227                        if (line.startsWith("# JOSM TagChecker")) {
228                            tagcheckerfile = true;
229                        }
230                        if (line.startsWith("# JOSM IgnoreTags")) {
231                            ignorefile = true;
232                        }
233                        continue;
234                    } else if (ignorefile) {
235                        line = line.trim();
236                        if (line.length() < 4) {
237                            continue;
238                        }
239
240                        String key = line.substring(0, 2);
241                        line = line.substring(2);
242
243                        if (key.equals("S:")) {
244                            ignoreDataStartsWith.add(line);
245                        } else if (key.equals("E:")) {
246                            ignoreDataEquals.add(line);
247                        } else if (key.equals("F:")) {
248                            ignoreDataEndsWith.add(line);
249                        } else if (key.equals("K:")) {
250                            IgnoreKeyPair tmp = new IgnoreKeyPair();
251                            int mid = line.indexOf('=');
252                            tmp.key = line.substring(0, mid);
253                            tmp.value = line.substring(mid+1);
254                            ignoreDataKeyPair.add(tmp);
255                        }
256                        continue;
257                    } else if (tagcheckerfile) {
258                        if (line.length() > 0) {
259                            CheckerData d = new CheckerData();
260                            String err = d.getData(line);
261
262                            if (err == null) {
263                                checkerData.add(d);
264                            } else {
265                                Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
266                            }
267                        }
268                    } else if (line.charAt(0) == '+') {
269                        okValue = line.substring(1);
270                    } else if (line.charAt(0) == '-' && okValue != null) {
271                        spellCheckKeyData.put(line.substring(1), okValue);
272                    } else {
273                        Main.error(tr("Invalid spellcheck line: {0}", line));
274                    }
275                }
276            } catch (IOException e) {
277                errorSources += source + "\n";
278            } finally {
279                Utils.close(reader);
280            }
281        }
282
283        if (errorSources.length() > 0)
284            throw new IOException( tr("Could not access data file(s):\n{0}", errorSources) );
285    }
286
287    /**
288     * Reads the presets data.
289     *
290     */
291    public static void initializePresets() {
292
293        if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
294            return;
295
296        Collection<TaggingPreset> presets = TaggingPresetPreference.taggingPresets;
297        if (presets != null) {
298            presetsValueData = new MultiMap<String, String>();
299            for (String a : OsmPrimitive.getUninterestingKeys()) {
300                presetsValueData.putVoid(a);
301            }
302            // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
303            /*  for(String a : OsmPrimitive.getDirectionKeys())
304                presetsValueData.add(a);
305             */
306            for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
307                    Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
308                presetsValueData.putVoid(a);
309            }
310            for (TaggingPreset p : presets) {
311                for (TaggingPresetItem i : p.data) {
312                    if (i instanceof KeyedItem) {
313                        addPresetValue(p, (KeyedItem) i);
314                    } else if (i instanceof CheckGroup) {
315                        for (Check c : ((CheckGroup) i).checks) {
316                            addPresetValue(p, c);
317                        }
318                    }
319                }
320            }
321        }
322    }
323
324    private static void addPresetValue(TaggingPreset p, KeyedItem ky) {
325        if (ky.key != null && ky.getValues() != null) {
326            try {
327                presetsValueData.putAll(ky.key, ky.getValues());
328            } catch (NullPointerException e) {
329                Main.error(p+": Unable to initialize "+ky);
330            }
331        }
332    }
333
334    @Override
335    public void visit(Node n) {
336        checkPrimitive(n);
337    }
338
339    @Override
340    public void visit(Relation n) {
341        checkPrimitive(n);
342    }
343
344    @Override
345    public void visit(Way w) {
346        checkPrimitive(w);
347    }
348
349    /**
350     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
351     * @param s string to check
352     */
353    private boolean containsLow(String s) {
354        if (s == null)
355            return false;
356        for (int i = 0; i < s.length(); i++) {
357            if (s.charAt(i) < 0x20)
358                return true;
359        }
360        return false;
361    }
362
363    /**
364     * Checks the primitive tags
365     * @param p The primitive to check
366     */
367    private void checkPrimitive(OsmPrimitive p) {
368        // Just a collection to know if a primitive has been already marked with error
369        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<OsmPrimitive, String>();
370
371        if (checkComplex) {
372            Map<String, String> keys = p.getKeys();
373            for (CheckerData d : checkerData) {
374                if (d.match(p, keys)) {
375                    errors.add( new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"),
376                            d.getDescription(), d.getDescriptionOrig(), d.getCode(), p) );
377                    withErrors.put(p, "TC");
378                }
379            }
380        }
381
382        for (Entry<String, String> prop : p.getKeys().entrySet()) {
383            String s = marktr("Key ''{0}'' invalid.");
384            String key = prop.getKey();
385            String value = prop.getValue();
386            if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
387                errors.add( new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"),
388                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p) );
389                withErrors.put(p, "ICV");
390            }
391            if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
392                errors.add( new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"),
393                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p) );
394                withErrors.put(p, "ICK");
395            }
396            if (checkValues && (value!=null && value.length() > 255) && !withErrors.contains(p, "LV")) {
397                errors.add( new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"),
398                        tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p) );
399                withErrors.put(p, "LV");
400            }
401            if (checkKeys && (key!=null && key.length() > 255) && !withErrors.contains(p, "LK")) {
402                errors.add( new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"),
403                        tr(s, key), MessageFormat.format(s, key), LONG_KEY, p) );
404                withErrors.put(p, "LK");
405            }
406            if (checkValues && (value==null || value.trim().length() == 0) && !withErrors.contains(p, "EV")) {
407                errors.add( new TestError(this, Severity.WARNING, tr("Tags with empty values"),
408                        tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p) );
409                withErrors.put(p, "EV");
410            }
411            if (checkKeys && spellCheckKeyData.containsKey(key) && !withErrors.contains(p, "IPK")) {
412                errors.add( new TestError(this, Severity.WARNING, tr("Invalid property key"),
413                        tr(s, key), MessageFormat.format(s, key), INVALID_KEY, p) );
414                withErrors.put(p, "IPK");
415            }
416            if (checkKeys && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
417                errors.add( new TestError(this, Severity.WARNING, tr("Invalid white space in property key"),
418                        tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p) );
419                withErrors.put(p, "IPK");
420            }
421            if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
422                errors.add( new TestError(this, Severity.OTHER, tr("Property values start or end with white space"),
423                        tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p) );
424                withErrors.put(p, "SPACE");
425            }
426            if (checkValues && value != null && !value.equals(entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
427                errors.add( new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"),
428                        tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p) );
429                withErrors.put(p, "HTML");
430            }
431            if (checkValues && value != null && value.length() > 0 && presetsValueData != null) {
432                final Set<String> values = presetsValueData.get(key);
433                final boolean keyInPresets = values != null;
434                final boolean tagInPresets = values != null && (values.isEmpty() || values.contains(prop.getValue()));
435
436                boolean ignore = false;
437                for (String a : ignoreDataStartsWith) {
438                    if (key.startsWith(a)) {
439                        ignore = true;
440                    }
441                }
442                for (String a : ignoreDataEquals) {
443                    if(key.equals(a)) {
444                        ignore = true;
445                    }
446                }
447                for (String a : ignoreDataEndsWith) {
448                    if(key.endsWith(a)) {
449                        ignore = true;
450                    }
451                }
452
453                if (!tagInPresets) {
454                    for (IgnoreKeyPair a : ignoreDataKeyPair) {
455                        if (key.equals(a.key) && value.equals(a.value)) {
456                            ignore = true;
457                        }
458                    }
459                }
460
461                if (!ignore) {
462                    if (!keyInPresets) {
463                        String i = marktr("Key ''{0}'' not in presets.");
464                        errors.add( new TestError(this, Severity.OTHER, tr("Presets do not contain property key"),
465                                tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p) );
466                        withErrors.put(p, "UPK");
467                    } else if (!tagInPresets) {
468                        String i = marktr("Value ''{0}'' for key ''{1}'' not in presets.");
469                        errors.add( new TestError(this, Severity.OTHER, tr("Presets do not contain property value"),
470                                tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p) );
471                        withErrors.put(p, "UPV");
472                    }
473                }
474            }
475            if (checkFixmes && value != null && value.length() > 0) {
476                if ((value.toLowerCase().contains("fixme")
477                        || value.contains("check and delete")
478                        || key.contains("todo") || key.toLowerCase().contains("fixme"))
479                        && !withErrors.contains(p, "FIXME")) {
480                    errors.add(new TestError(this, Severity.OTHER,
481                            tr("FIXMES"), FIXME, p));
482                    withErrors.put(p, "FIXME");
483                }
484            }
485        }
486    }
487
488    @Override
489    public void startTest(ProgressMonitor monitor) {
490        super.startTest(monitor);
491        checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
492        if (isBeforeUpload) {
493            checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
494        }
495
496        checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
497        if (isBeforeUpload) {
498            checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
499        }
500
501        checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
502        if (isBeforeUpload) {
503            checkComplex = checkValues && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
504        }
505
506        checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
507        if (isBeforeUpload) {
508            checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
509        }
510    }
511
512    @Override
513    public void visit(Collection<OsmPrimitive> selection) {
514        if (checkKeys || checkValues || checkComplex || checkFixmes) {
515            super.visit(selection);
516        }
517    }
518
519    @Override
520    public void addGui(JPanel testPanel) {
521        GBC a = GBC.eol();
522        a.anchor = GridBagConstraints.EAST;
523
524        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3,0,0,0));
525
526        prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
527        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
528        testPanel.add(prefCheckKeys, GBC.std().insets(20,0,0,0));
529
530        prefCheckKeysBeforeUpload = new JCheckBox();
531        prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
532        testPanel.add(prefCheckKeysBeforeUpload, a);
533
534        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
535        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
536        testPanel.add(prefCheckComplex, GBC.std().insets(20,0,0,0));
537
538        prefCheckComplexBeforeUpload = new JCheckBox();
539        prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
540        testPanel.add(prefCheckComplexBeforeUpload, a);
541
542        sourcesList = new JList(new DefaultListModel());
543
544        String sources = Main.pref.get( PREF_SOURCES );
545        if (sources != null && sources.length() > 0) {
546            for (String source : sources.split(";")) {
547                ((DefaultListModel)sourcesList.getModel()).addElement(source);
548            }
549        }
550
551        addSrcButton = new JButton(tr("Add"));
552        addSrcButton.addActionListener(new ActionListener() {
553            @Override
554            public void actionPerformed(ActionEvent e) {
555                String source = JOptionPane.showInputDialog(
556                        Main.parent,
557                        tr("TagChecker source"),
558                        tr("TagChecker source"),
559                        JOptionPane.QUESTION_MESSAGE);
560                if (source != null) {
561                    ((DefaultListModel)sourcesList.getModel()).addElement(source);
562                }
563                sourcesList.clearSelection();
564            }
565        });
566
567        editSrcButton = new JButton(tr("Edit"));
568        editSrcButton.addActionListener(new ActionListener() {
569            @Override
570            public void actionPerformed(ActionEvent e) {
571                int row = sourcesList.getSelectedIndex();
572                if (row == -1 && sourcesList.getModel().getSize() == 1) {
573                    sourcesList.setSelectedIndex(0);
574                    row = 0;
575                }
576                if (row == -1) {
577                    if (sourcesList.getModel().getSize() == 0) {
578                        String source = JOptionPane.showInputDialog(Main.parent, tr("TagChecker source"), tr("TagChecker source"), JOptionPane.QUESTION_MESSAGE);
579                        if (source != null) {
580                            ((DefaultListModel)sourcesList.getModel()).addElement(source);
581                        }
582                    } else {
583                        JOptionPane.showMessageDialog(
584                                Main.parent,
585                                tr("Please select the row to edit."),
586                                tr("Information"),
587                                JOptionPane.INFORMATION_MESSAGE
588                                );
589                    }
590                } else {
591                    String source = (String)JOptionPane.showInputDialog(Main.parent,
592                            tr("TagChecker source"),
593                            tr("TagChecker source"),
594                            JOptionPane.QUESTION_MESSAGE, null, null,
595                            sourcesList.getSelectedValue());
596                    if (source != null) {
597                        ((DefaultListModel)sourcesList.getModel()).setElementAt(source, row);
598                    }
599                }
600                sourcesList.clearSelection();
601            }
602        });
603
604        deleteSrcButton = new JButton(tr("Delete"));
605        deleteSrcButton.addActionListener(new ActionListener() {
606            @Override
607            public void actionPerformed(ActionEvent e) {
608                if (sourcesList.getSelectedIndex() == -1) {
609                    JOptionPane.showMessageDialog(Main.parent, tr("Please select the row to delete."), tr("Information"), JOptionPane.QUESTION_MESSAGE);
610                } else {
611                    ((DefaultListModel)sourcesList.getModel()).remove(sourcesList.getSelectedIndex());
612                }
613            }
614        });
615        sourcesList.setMinimumSize(new Dimension(300,50));
616        sourcesList.setVisibleRowCount(3);
617
618        sourcesList.setToolTipText(tr("The sources (URL or filename) of spell check (see http://wiki.openstreetmap.org/index.php/User:JLS/speller) or tag checking data files."));
619        addSrcButton.setToolTipText(tr("Add a new source to the list."));
620        editSrcButton.setToolTipText(tr("Edit the selected source."));
621        deleteSrcButton.setToolTipText(tr("Delete the selected source from the list."));
622
623        testPanel.add(new JLabel(tr("Data sources")), GBC.eol().insets(23,0,0,0));
624        testPanel.add(new JScrollPane(sourcesList), GBC.eol().insets(23,0,0,0).fill(GridBagConstraints.HORIZONTAL));
625        final JPanel buttonPanel = new JPanel(new GridBagLayout());
626        testPanel.add(buttonPanel, GBC.eol().fill(GridBagConstraints.HORIZONTAL));
627        buttonPanel.add(addSrcButton, GBC.std().insets(0,5,0,0));
628        buttonPanel.add(editSrcButton, GBC.std().insets(5,5,5,0));
629        buttonPanel.add(deleteSrcButton, GBC.std().insets(0,5,0,0));
630
631        ActionListener disableCheckActionListener = new ActionListener() {
632            @Override
633            public void actionPerformed(ActionEvent e) {
634                handlePrefEnable();
635            }
636        };
637        prefCheckKeys.addActionListener(disableCheckActionListener);
638        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
639        prefCheckComplex.addActionListener(disableCheckActionListener);
640        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
641
642        handlePrefEnable();
643
644        prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
645        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
646        testPanel.add(prefCheckValues, GBC.std().insets(20,0,0,0));
647
648        prefCheckValuesBeforeUpload = new JCheckBox();
649        prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
650        testPanel.add(prefCheckValuesBeforeUpload, a);
651
652        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
653        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
654        testPanel.add(prefCheckFixmes, GBC.std().insets(20,0,0,0));
655
656        prefCheckFixmesBeforeUpload = new JCheckBox();
657        prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
658        testPanel.add(prefCheckFixmesBeforeUpload, a);
659
660        prefUseDataFile = new JCheckBox(tr("Use default data file."), Main.pref.getBoolean(PREF_USE_DATA_FILE, true));
661        prefUseDataFile.setToolTipText(tr("Use the default data file (recommended)."));
662        testPanel.add(prefUseDataFile, GBC.eol().insets(20,0,0,0));
663
664        prefUseIgnoreFile = new JCheckBox(tr("Use default tag ignore file."), Main.pref.getBoolean(PREF_USE_IGNORE_FILE, true));
665        prefUseIgnoreFile.setToolTipText(tr("Use the default tag ignore file (recommended)."));
666        testPanel.add(prefUseIgnoreFile, GBC.eol().insets(20,0,0,0));
667
668        prefUseSpellFile = new JCheckBox(tr("Use default spellcheck file."), Main.pref.getBoolean(PREF_USE_SPELL_FILE, true));
669        prefUseSpellFile.setToolTipText(tr("Use the default spellcheck file (recommended)."));
670        testPanel.add(prefUseSpellFile, GBC.eol().insets(20,0,0,0));
671    }
672
673    public void handlePrefEnable() {
674        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
675                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
676        sourcesList.setEnabled( selected );
677        addSrcButton.setEnabled(selected);
678        editSrcButton.setEnabled(selected);
679        deleteSrcButton.setEnabled(selected);
680    }
681
682    @Override
683    public boolean ok() {
684        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
685        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
686                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
687
688        Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
689        Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
690        Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
691        Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
692        Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
693        Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
694        Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
695        Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
696        Main.pref.put(PREF_USE_DATA_FILE, prefUseDataFile.isSelected());
697        Main.pref.put(PREF_USE_IGNORE_FILE, prefUseIgnoreFile.isSelected());
698        Main.pref.put(PREF_USE_SPELL_FILE, prefUseSpellFile.isSelected());
699        StringBuilder sources = new StringBuilder();
700        if (sourcesList.getModel().getSize() > 0) {
701            for (int i = 0; i < sourcesList.getModel().getSize(); ++i) {
702                if (sources.length() > 0) {
703                    sources.append(";");
704                }
705                sources.append(sourcesList.getModel().getElementAt(i));
706            }
707        }
708        return Main.pref.put(PREF_SOURCES, sources.length() > 0 ? sources.toString() : null);
709    }
710
711    @Override
712    public Command fixError(TestError testError) {
713
714        List<Command> commands = new ArrayList<Command>(50);
715
716        Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
717        for (OsmPrimitive p : primitives) {
718            Map<String, String> tags = p.getKeys();
719            if (tags == null || tags.isEmpty()) {
720                continue;
721            }
722
723            for (Entry<String, String> prop: tags.entrySet()) {
724                String key = prop.getKey();
725                String value = prop.getValue();
726                if (value == null || value.trim().length() == 0) {
727                    commands.add(new ChangePropertyCommand(p, key, null));
728                } else if (value.startsWith(" ") || value.endsWith(" ")) {
729                    commands.add(new ChangePropertyCommand(p, key, value.trim()));
730                } else if (key.startsWith(" ") || key.endsWith(" ")) {
731                    commands.add(new ChangePropertyKeyCommand(p, key, key.trim()));
732                } else {
733                    String evalue = entities.unescape(value);
734                    if (!evalue.equals(value)) {
735                        commands.add(new ChangePropertyCommand(p, key, evalue));
736                    } else {
737                        String replacementKey = spellCheckKeyData.get(key);
738                        if (replacementKey != null) {
739                            commands.add(new ChangePropertyKeyCommand(p, key, replacementKey));
740                        }
741                    }
742                }
743            }
744        }
745
746        if (commands.isEmpty())
747            return null;
748        if (commands.size() == 1)
749            return commands.get(0);
750
751        return new SequenceCommand(tr("Fix tags"), commands);
752    }
753
754    @Override
755    public boolean isFixable(TestError testError) {
756
757        if (testError.getTester() instanceof TagChecker) {
758            int code = testError.getCode();
759            return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || code == INVALID_KEY_SPACE || code == INVALID_HTML;
760        }
761
762        return false;
763    }
764
765    protected static class IgnoreKeyPair {
766        public String key;
767        public String value;
768    }
769
770    protected static class CheckerData {
771        private String description;
772        protected List<CheckerElement> data = new ArrayList<CheckerElement>();
773        private OsmPrimitiveType type;
774        private int code;
775        protected Severity severity;
776        protected static final int TAG_CHECK_ERROR  = 1250;
777        protected static final int TAG_CHECK_WARN   = 1260;
778        protected static final int TAG_CHECK_INFO   = 1270;
779
780        protected static class CheckerElement {
781            public Object tag;
782            public Object value;
783            public boolean noMatch;
784            public boolean tagAll = false;
785            public boolean valueAll = false;
786            public boolean valueBool = false;
787
788            private Pattern getPattern(String str) throws IllegalStateException, PatternSyntaxException {
789                if (str.endsWith("/i"))
790                    return Pattern.compile(str.substring(1,str.length()-2), Pattern.CASE_INSENSITIVE);
791                if (str.endsWith("/"))
792                    return Pattern.compile(str.substring(1,str.length()-1));
793
794                throw new IllegalStateException();
795            }
796            public CheckerElement(String exp) throws IllegalStateException, PatternSyntaxException {
797                Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
798                m.matches();
799
800                String n = m.group(1).trim();
801
802                if(n.equals("*")) {
803                    tagAll = true;
804                } else {
805                    tag = n.startsWith("/") ? getPattern(n) : n;
806                    noMatch = m.group(2).equals("!=");
807                    n = m.group(3).trim();
808                    if (n.equals("*")) {
809                        valueAll = true;
810                    } else if (n.equals("BOOLEAN_TRUE")) {
811                        valueBool = true;
812                        value = OsmUtils.trueval;
813                    } else if (n.equals("BOOLEAN_FALSE")) {
814                        valueBool = true;
815                        value = OsmUtils.falseval;
816                    } else {
817                        value = n.startsWith("/") ? getPattern(n) : n;
818                    }
819                }
820            }
821
822            public boolean match(OsmPrimitive osm, Map<String, String> keys) {
823                for (Entry<String, String> prop: keys.entrySet()) {
824                    String key = prop.getKey();
825                    String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
826                    if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
827                            && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
828                        return !noMatch;
829                }
830                return noMatch;
831            }
832        }
833
834        public String getData(String str) {
835            Matcher m = Pattern.compile(" *# *([^#]+) *$").matcher(str);
836            str = m.replaceFirst("").trim();
837            try {
838                description = m.group(1);
839                if (description != null && description.length() == 0) {
840                    description = null;
841                }
842            } catch (IllegalStateException e) {
843                description = null;
844            }
845            String[] n = str.split(" *: *", 3);
846            if (n[0].equals("way")) {
847                type = OsmPrimitiveType.WAY;
848            } else if (n[0].equals("node")) {
849                type = OsmPrimitiveType.NODE;
850            } else if (n[0].equals("relation")) {
851                type = OsmPrimitiveType.RELATION;
852            } else if (n[0].equals("*")) {
853                type = null;
854            } else
855                return tr("Could not find element type");
856            if (n.length != 3)
857                return tr("Incorrect number of parameters");
858
859            if (n[1].equals("W")) {
860                severity = Severity.WARNING;
861                code = TAG_CHECK_WARN;
862            } else if (n[1].equals("E")) {
863                severity = Severity.ERROR;
864                code = TAG_CHECK_ERROR;
865            } else if(n[1].equals("I")) {
866                severity = Severity.OTHER;
867                code = TAG_CHECK_INFO;
868            } else
869                return tr("Could not find warning level");
870            for (String exp: n[2].split(" *&& *")) {
871                try {
872                    data.add(new CheckerElement(exp));
873                } catch (IllegalStateException e) {
874                    return tr("Illegal expression ''{0}''", exp);
875                }
876                catch (PatternSyntaxException e) {
877                    return tr("Illegal regular expression ''{0}''", exp);
878                }
879            }
880            return null;
881        }
882
883        public boolean match(OsmPrimitive osm, Map<String, String> keys) {
884            if (type != null && OsmPrimitiveType.from(osm) != type)
885                return false;
886
887            for (CheckerElement ce : data) {
888                if (!ce.match(osm, keys))
889                    return false;
890            }
891            return true;
892        }
893
894        public String getDescription() {
895            return tr(description);
896        }
897
898        public String getDescriptionOrig() {
899            return description;
900        }
901
902        public Severity getSeverity() {
903            return severity;
904        }
905
906        public int getCode() {
907            if (type == null)
908                return code;
909
910            return code + type.ordinal() + 1;
911        }
912    }
913}