001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004
005import java.awt.Graphics2D;
006import java.awt.image.BufferedImage;
007import java.io.BufferedOutputStream;
008import java.io.File;
009import java.io.FileInputStream;
010import java.io.FileNotFoundException;
011import java.io.FileOutputStream;
012import java.io.IOException;
013import java.io.InputStream;
014import java.io.OutputStream;
015import java.lang.ref.SoftReference;
016import java.net.URLConnection;
017import java.text.SimpleDateFormat;
018import java.util.ArrayList;
019import java.util.Calendar;
020import java.util.Collections;
021import java.util.Comparator;
022import java.util.HashMap;
023import java.util.HashSet;
024import java.util.Iterator;
025import java.util.List;
026import java.util.Map;
027import java.util.Properties;
028import java.util.Set;
029
030import javax.imageio.ImageIO;
031import javax.xml.bind.JAXBContext;
032import javax.xml.bind.Marshaller;
033import javax.xml.bind.Unmarshaller;
034
035import org.openstreetmap.josm.Main;
036import org.openstreetmap.josm.data.ProjectionBounds;
037import org.openstreetmap.josm.data.coor.EastNorth;
038import org.openstreetmap.josm.data.coor.LatLon;
039import org.openstreetmap.josm.data.imagery.types.EntryType;
040import org.openstreetmap.josm.data.imagery.types.ProjectionType;
041import org.openstreetmap.josm.data.imagery.types.WmsCacheType;
042import org.openstreetmap.josm.data.preferences.StringProperty;
043import org.openstreetmap.josm.data.projection.Projection;
044import org.openstreetmap.josm.gui.NavigatableComponent;
045import org.openstreetmap.josm.tools.Utils;
046
047
048
049public class WmsCache {
050    //TODO Property for maximum cache size
051    //TODO Property for maximum age of tile, automatically remove old tiles
052    //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache
053    //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load
054
055    private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms");
056    private static final String INDEX_FILENAME = "index.xml";
057    private static final String LAYERS_INDEX_FILENAME = "layers.properties";
058
059    private static class CacheEntry {
060        final double pixelPerDegree;
061        final double east;
062        final double north;
063        final ProjectionBounds bounds;
064        final String filename;
065
066        long lastUsed;
067        long lastModified;
068
069        CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) {
070            this.pixelPerDegree = pixelPerDegree;
071            this.east = east;
072            this.north = north;
073            this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree);
074            this.filename = filename;
075        }
076    }
077
078    private static class ProjectionEntries {
079        final String projection;
080        final String cacheDirectory;
081        final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>();
082
083        ProjectionEntries(String projection, String cacheDirectory) {
084            this.projection = projection;
085            this.cacheDirectory = cacheDirectory;
086        }
087    }
088
089    private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>();
090    private final File cacheDir;
091    private final int tileSize; // Should be always 500
092    private int totalFileSize;
093    private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated
094    // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found
095    private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
096    private Set<ProjectionBounds> areaToCache;
097
098    protected String cacheDirPath() {
099        String cPath = PROP_CACHE_PATH.get();
100        if (!(new File(cPath).isAbsolute())) {
101            cPath = Main.pref.getCacheDirectory() + File.separator + cPath;
102        }
103        return cPath;
104    }
105
106    public WmsCache(String url, int tileSize) {
107        File globalCacheDir = new File(cacheDirPath());
108        globalCacheDir.mkdirs();
109        cacheDir = new File(globalCacheDir, getCacheDirectory(url));
110        cacheDir.mkdirs();
111        this.tileSize = tileSize;
112    }
113
114    private String getCacheDirectory(String url) {
115        String cacheDirName = null;
116        InputStream fis = null;
117        OutputStream fos = null;
118        try {
119            Properties layersIndex = new Properties();
120            File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME);
121            try {
122                fis = new FileInputStream(layerIndexFile);
123                layersIndex.load(fis);
124            } catch (FileNotFoundException e) {
125                Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)");
126            } catch (IOException e) {
127                Main.error("Unable to load layers index for wms cache");
128                e.printStackTrace();
129            }
130
131            for (Object propKey: layersIndex.keySet()) {
132                String s = (String)propKey;
133                if (url.equals(layersIndex.getProperty(s))) {
134                    cacheDirName = s;
135                    break;
136                }
137            }
138
139            if (cacheDirName == null) {
140                int counter = 0;
141                while (true) {
142                    counter++;
143                    if (!layersIndex.keySet().contains(String.valueOf(counter))) {
144                        break;
145                    }
146                }
147                cacheDirName = String.valueOf(counter);
148                layersIndex.setProperty(cacheDirName, url);
149                try {
150                    fos = new FileOutputStream(layerIndexFile);
151                    layersIndex.store(fos, "");
152                } catch (IOException e) {
153                    Main.error("Unable to save layer index for wms cache");
154                    e.printStackTrace();
155                }
156            }
157        } finally {
158            Utils.close(fos);
159            Utils.close(fis);
160        }
161
162        return cacheDirName;
163    }
164
165    private ProjectionEntries getProjectionEntries(Projection projection) {
166        return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName());
167    }
168
169    private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) {
170        ProjectionEntries result = entries.get(projection);
171        if (result == null) {
172            result = new ProjectionEntries(projection, cacheDirectory);
173            entries.put(projection, result);
174        }
175
176        return result;
177    }
178
179    public synchronized void loadIndex() {
180        File indexFile = new File(cacheDir, INDEX_FILENAME);
181        try {
182            JAXBContext context = JAXBContext.newInstance(
183                    WmsCacheType.class.getPackage().getName(),
184                    WmsCacheType.class.getClassLoader());
185            Unmarshaller unmarshaller = context.createUnmarshaller();
186            WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile));
187            totalFileSize = cacheEntries.getTotalFileSize();
188            if (cacheEntries.getTileSize() != tileSize) {
189                Main.info("Cache created with different tileSize, cache will be discarded");
190                return;
191            }
192            for (ProjectionType projectionType: cacheEntries.getProjection()) {
193                ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory());
194                for (EntryType entry: projectionType.getEntry()) {
195                    CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename());
196                    ce.lastUsed = entry.getLastUsed().getTimeInMillis();
197                    ce.lastModified = entry.getLastModified().getTimeInMillis();
198                    projection.entries.add(ce);
199                }
200            }
201        } catch (Exception e) {
202            if (indexFile.exists()) {
203                e.printStackTrace();
204                Main.info("Unable to load index for wms-cache, new file will be created");
205            } else {
206                Main.info("Index for wms-cache doesn't exist, new file will be created");
207            }
208        }
209
210        removeNonReferencedFiles();
211    }
212
213    private void removeNonReferencedFiles() {
214
215        Set<String> usedProjections = new HashSet<String>();
216
217        for (ProjectionEntries projectionEntries: entries.values()) {
218
219            usedProjections.add(projectionEntries.cacheDirectory);
220
221            File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory);
222            if (projectionDir.exists()) {
223                Set<String> referencedFiles = new HashSet<String>();
224
225                for (CacheEntry ce: projectionEntries.entries) {
226                    referencedFiles.add(ce.filename);
227                }
228
229                for (File file: projectionDir.listFiles()) {
230                    if (!referencedFiles.contains(file.getName())) {
231                        file.delete();
232                    }
233                }
234            }
235        }
236
237        for (File projectionDir: cacheDir.listFiles()) {
238            if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) {
239                Utils.deleteDirectory(projectionDir);
240            }
241        }
242    }
243
244    private int calculateTotalFileSize() {
245        int result = 0;
246        for (ProjectionEntries projectionEntries: entries.values()) {
247            Iterator<CacheEntry> it = projectionEntries.entries.iterator();
248            while (it.hasNext()) {
249                CacheEntry entry = it.next();
250                File imageFile = getImageFile(projectionEntries, entry);
251                if (!imageFile.exists()) {
252                    it.remove();
253                } else {
254                    result += imageFile.length();
255                }
256            }
257        }
258        return result;
259    }
260
261    public synchronized void saveIndex() {
262        WmsCacheType index = new WmsCacheType();
263
264        if (totalFileSizeDirty) {
265            totalFileSize = calculateTotalFileSize();
266        }
267
268        index.setTileSize(tileSize);
269        index.setTotalFileSize(totalFileSize);
270        for (ProjectionEntries projectionEntries: entries.values()) {
271            if (!projectionEntries.entries.isEmpty()) {
272                ProjectionType projectionType = new ProjectionType();
273                projectionType.setName(projectionEntries.projection);
274                projectionType.setCacheDirectory(projectionEntries.cacheDirectory);
275                index.getProjection().add(projectionType);
276                for (CacheEntry ce: projectionEntries.entries) {
277                    EntryType entry = new EntryType();
278                    entry.setPixelPerDegree(ce.pixelPerDegree);
279                    entry.setEast(ce.east);
280                    entry.setNorth(ce.north);
281                    Calendar c = Calendar.getInstance();
282                    c.setTimeInMillis(ce.lastUsed);
283                    entry.setLastUsed(c);
284                    c = Calendar.getInstance();
285                    c.setTimeInMillis(ce.lastModified);
286                    entry.setLastModified(c);
287                    entry.setFilename(ce.filename);
288                    projectionType.getEntry().add(entry);
289                }
290            }
291        }
292        try {
293            JAXBContext context = JAXBContext.newInstance(
294                    WmsCacheType.class.getPackage().getName(),
295                    WmsCacheType.class.getClassLoader());
296            Marshaller marshaller = context.createMarshaller();
297            marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME)));
298        } catch (Exception e) {
299            Main.error("Failed to save wms-cache file");
300            e.printStackTrace();
301        }
302    }
303
304    private File getImageFile(ProjectionEntries projection, CacheEntry entry) {
305        return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename);
306    }
307
308
309    private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException {
310
311        synchronized (this) {
312            entry.lastUsed = System.currentTimeMillis();
313
314            SoftReference<BufferedImage> memCache = memoryCache.get(entry);
315            if (memCache != null) {
316                BufferedImage result = memCache.get();
317                if (result != null)
318                    return result;
319            }
320        }
321
322        try {
323            // Reading can't be in synchronized section, it's too slow
324            BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry));
325            synchronized (this) {
326                if (result == null) {
327                    projectionEntries.entries.remove(entry);
328                    totalFileSizeDirty = true;
329                }
330                return result;
331            }
332        } catch (IOException e) {
333            synchronized (this) {
334                projectionEntries.entries.remove(entry);
335                totalFileSizeDirty = true;
336                throw e;
337            }
338        }
339    }
340
341    private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) {
342        for (CacheEntry entry: projectionEntries.entries) {
343            if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north)
344                return entry;
345        }
346        return null;
347    }
348
349    public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
350        ProjectionEntries projectionEntries = getProjectionEntries(projection);
351        CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
352        return (entry != null);
353    }
354
355    public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) {
356        CacheEntry entry = null;
357        ProjectionEntries projectionEntries = null;
358        synchronized (this) {
359            projectionEntries = getProjectionEntries(projection);
360            entry = findEntry(projectionEntries, pixelPerDegree, east, north);
361        }
362        if (entry != null) {
363            try {
364                return loadImage(projectionEntries, entry);
365            } catch (IOException e) {
366                Main.error("Unable to load file from wms cache");
367                e.printStackTrace();
368                return null;
369            }
370        }
371        return null;
372    }
373
374    public  BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) {
375        ProjectionEntries projectionEntries;
376        List<CacheEntry> matches;
377        synchronized (this) {
378            matches = new ArrayList<WmsCache.CacheEntry>();
379
380            double minPPD = pixelPerDegree / 5;
381            double maxPPD = pixelPerDegree * 5;
382            projectionEntries = getProjectionEntries(projection);
383
384            double size2 = tileSize / pixelPerDegree;
385            double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly
386            ProjectionBounds bounds = new ProjectionBounds(east + border, north + border,
387                    east + size2 - border, north + size2 - border);
388
389            //TODO Do not load tile if it is completely overlapped by other tile with better ppd
390            for (CacheEntry entry: projectionEntries.entries) {
391                if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) {
392                    entry.lastUsed = System.currentTimeMillis();
393                    matches.add(entry);
394                }
395            }
396
397            if (matches.isEmpty())
398                return null;
399
400
401            Collections.sort(matches, new Comparator<CacheEntry>() {
402                @Override
403                public int compare(CacheEntry o1, CacheEntry o2) {
404                    return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree);
405                }
406            });
407        }
408
409        //TODO Use alpha layer only when enabled on wms layer
410        BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR);
411        Graphics2D g = result.createGraphics();
412
413
414        boolean drawAtLeastOnce = false;
415        Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>();
416        for (CacheEntry ce: matches) {
417            BufferedImage img;
418            try {
419                img = loadImage(projectionEntries, ce);
420                localCache.put(ce, new SoftReference<BufferedImage>(img));
421            } catch (IOException e) {
422                continue;
423            }
424
425            drawAtLeastOnce = true;
426
427            int xDiff = (int)((ce.east - east) * pixelPerDegree);
428            int yDiff = (int)((ce.north - north) * pixelPerDegree);
429            int size = (int)(pixelPerDegree / ce.pixelPerDegree  * tileSize);
430
431            int x = xDiff;
432            int y = -size + tileSize - yDiff;
433
434            g.drawImage(img, x, y, size, size, null);
435        }
436
437        if (drawAtLeastOnce) {
438            synchronized (this) {
439                memoryCache.putAll(localCache);
440            }
441            return result;
442        } else
443            return null;
444    }
445
446    private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) {
447        LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north));
448        LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north));
449        LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree));
450
451        double deltaLat = Math.abs(ll3.lat() - ll1.lat());
452        double deltaLon = Math.abs(ll3.lon() - ll1.lon());
453        int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1);
454        int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1);
455
456        String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2));
457        String extension;
458        if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) {
459            extension = "jpg";
460        } else if ("image/png".equals(mimeType)) {
461            extension = "png";
462        } else if ("image/gif".equals(mimeType)) {
463            extension = "gif";
464        } else {
465            extension = "dat";
466        }
467
468        int counter = 0;
469        FILENAME_LOOP:
470            while (true) {
471                String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension);
472                for (CacheEntry entry: projectionEntries.entries) {
473                    if (entry.filename.equals(result)) {
474                        counter++;
475                        continue FILENAME_LOOP;
476                    }
477                }
478                return result;
479            }
480    }
481
482    /**
483     *
484     * @param img Used only when overlapping is used, when not used, used raw from imageData
485     * @param imageData
486     * @param projection
487     * @param pixelPerDegree
488     * @param east
489     * @param north
490     * @throws IOException
491     */
492    public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException {
493        ProjectionEntries projectionEntries = getProjectionEntries(projection);
494        CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north);
495        File imageFile;
496        if (entry == null) {
497
498            String mimeType;
499            if (img != null) {
500                mimeType = "image/png";
501            } else {
502                mimeType = URLConnection.guessContentTypeFromStream(imageData);
503            }
504            entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType));
505            entry.lastUsed = System.currentTimeMillis();
506            entry.lastModified = entry.lastUsed;
507            projectionEntries.entries.add(entry);
508            imageFile = getImageFile(projectionEntries, entry);
509        } else {
510            imageFile = getImageFile(projectionEntries, entry);
511            totalFileSize -= imageFile.length();
512        }
513
514        imageFile.getParentFile().mkdirs();
515
516        if (img != null) {
517            BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType());
518            copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null);
519            ImageIO.write(copy, "png", imageFile);
520            totalFileSize += imageFile.length();
521        } else {
522            OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile));
523            try {
524                totalFileSize += Utils.copyStream(imageData, os);
525            } finally {
526                Utils.close(os);
527            }
528        }
529    }
530
531    public synchronized void cleanSmallFiles(int size) {
532        for (ProjectionEntries projectionEntries: entries.values()) {
533            Iterator<CacheEntry> it = projectionEntries.entries.iterator();
534            while (it.hasNext()) {
535                File file = getImageFile(projectionEntries, it.next());
536                long length = file.length();
537                if (length <= size) {
538                    if (length == 0) {
539                        totalFileSizeDirty = true; // File probably doesn't exist
540                    }
541                    totalFileSize -= size;
542                    file.delete();
543                    it.remove();
544                }
545            }
546        }
547    }
548
549    public static String printDate(Calendar c) {
550        return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime());
551    }
552
553    private boolean isInsideAreaToCache(CacheEntry cacheEntry) {
554        for (ProjectionBounds b: areaToCache) {
555            if (cacheEntry.bounds.intersects(b))
556                return true;
557        }
558        return false;
559    }
560
561    public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) {
562        this.areaToCache = areaToCache;
563        Iterator<CacheEntry> it = memoryCache.keySet().iterator();
564        while (it.hasNext()) {
565            if (!isInsideAreaToCache(it.next())) {
566                it.remove();
567            }
568        }
569    }
570}