001package org.openstreetmap.gui.jmapviewer;
002
003//License: GPL. Copyright 2008 by Jan Peter Stotz
004
005import java.io.BufferedReader;
006import java.io.ByteArrayInputStream;
007import java.io.ByteArrayOutputStream;
008import java.io.File;
009import java.io.FileInputStream;
010import java.io.FileNotFoundException;
011import java.io.FileOutputStream;
012import java.io.IOException;
013import java.io.InputStream;
014import java.io.InputStreamReader;
015import java.io.OutputStreamWriter;
016import java.io.PrintWriter;
017import java.net.HttpURLConnection;
018import java.net.URL;
019import java.net.URLConnection;
020import java.nio.charset.Charset;
021import java.util.HashMap;
022import java.util.Map;
023import java.util.Map.Entry;
024import java.util.Random;
025import java.util.logging.Level;
026import java.util.logging.Logger;
027
028import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader;
029import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController;
030import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
031import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
032import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
033import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
034import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate;
035
036/**
037 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and
038 * saves all loaded files in a directory located in the temporary directory.
039 * If a tile is present in this file cache it will not be loaded from OSM again.
040 *
041 * @author Jan Peter Stotz
042 * @author Stefan Zeller
043 */
044public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader {
045
046    private static final Logger log = Logger.getLogger(OsmFileCacheTileLoader.class.getName());
047
048    private static final String ETAG_FILE_EXT = ".etag";
049    private static final String TAGS_FILE_EXT = ".tags";
050
051    private static final Charset TAGS_CHARSET = Charset.forName("UTF-8");
052
053    public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24;
054    public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7;
055
056    protected String cacheDirBase;
057
058    protected final Map<TileSource, File> sourceCacheDirMap;
059
060    protected long maxCacheFileAge = FILE_AGE_ONE_WEEK;
061    protected long recheckAfter = FILE_AGE_ONE_DAY;
062
063    public static File getDefaultCacheDir() throws SecurityException {
064        String tempDir = null;
065        String userName = System.getProperty("user.name");
066        try {
067            tempDir = System.getProperty("java.io.tmpdir");
068        } catch (SecurityException e) {
069            log.log(Level.WARNING,
070                    "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: "
071                    + e.toString());
072            throw e; // rethrow
073        }
074        try {
075            if (tempDir == null)
076                throw new IOException("No temp directory set");
077            String subDirName = "JMapViewerTiles";
078            // On Linux/Unix systems we do not have a per user tmp directory.
079            // Therefore we add the user name for getting a unique dir name.
080            if (userName != null && userName.length() > 0) {
081                subDirName += "_" + userName;
082            }
083            File cacheDir = new File(tempDir, subDirName);
084            return cacheDir;
085        } catch (Exception e) {
086        }
087        return null;
088    }
089
090    /**
091     * Create a OSMFileCacheTileLoader with given cache directory.
092     * If cacheDir is not set or invalid, IOException will be thrown.
093     * @param map the listener checking for tile load events (usually the map for display)
094     * @param cacheDir directory to store cached tiles
095     */
096    public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException  {
097        super(map);
098        if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs()))
099            throw new IOException("Cannot access cache directory");
100
101        log.finest("Tile cache directory: " + cacheDir);
102        cacheDirBase = cacheDir.getAbsolutePath();
103        sourceCacheDirMap = new HashMap<TileSource, File>();
104    }
105
106    /**
107     * Create a OSMFileCacheTileLoader with system property temp dir.
108     * If not set an IOException will be thrown.
109     * @param map the listener checking for tile load events (usually the map for display)
110     */
111    public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException {
112        this(map, getDefaultCacheDir());
113    }
114
115    @Override
116    public TileJob createTileLoaderJob(final Tile tile) {
117        return new FileLoadJob(tile);
118    }
119
120    protected File getSourceCacheDir(TileSource source) {
121        File dir = sourceCacheDirMap.get(source);
122        if (dir == null) {
123            dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));
124            if (!dir.exists()) {
125                dir.mkdirs();
126            }
127        }
128        return dir;
129    }
130
131    protected class FileLoadJob implements TileJob {
132        InputStream input = null;
133
134        Tile tile;
135        File tileCacheDir;
136        File tileFile = null;
137        long fileAge = 0;
138        boolean fileTilePainted = false;
139
140        public FileLoadJob(Tile tile) {
141            this.tile = tile;
142        }
143
144        @Override
145        public Tile getTile() {
146            return tile;
147        }
148
149        @Override
150        public void run() {
151            synchronized (tile) {
152                if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading())
153                    return;
154                tile.loaded = false;
155                tile.error = false;
156                tile.loading = true;
157            }
158            tileCacheDir = getSourceCacheDir(tile.getSource());
159            if (loadTileFromFile()) {
160                return;
161            }
162            if (fileTilePainted) {
163                TileJob job = new TileJob() {
164
165                    @Override
166                    public void run() {
167                        loadOrUpdateTile();
168                    }
169                    @Override
170                    public Tile getTile() {
171                        return tile;
172                    }
173                };
174                JobDispatcher.getInstance().addJob(job);
175            } else {
176                loadOrUpdateTile();
177            }
178        }
179
180        protected void loadOrUpdateTile() {
181            try {
182                URLConnection urlConn = loadTileFromOsm(tile);
183                if (tileFile != null) {
184                    switch (tile.getSource().getTileUpdate()) {
185                    case IfModifiedSince:
186                        urlConn.setIfModifiedSince(fileAge);
187                        break;
188                    case LastModified:
189                        if (!isOsmTileNewer(fileAge)) {
190                            log.finest("LastModified test: local version is up to date: " + tile);
191                            tile.setLoaded(true);
192                            tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
193                            return;
194                        }
195                        break;
196                    }
197                }
198                if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) {
199                    String fileETag = tile.getValue("etag");
200                    if (fileETag != null) {
201                        switch (tile.getSource().getTileUpdate()) {
202                        case IfNoneMatch:
203                            urlConn.addRequestProperty("If-None-Match", fileETag);
204                            break;
205                        case ETag:
206                            if (hasOsmTileETag(fileETag)) {
207                                tile.setLoaded(true);
208                                tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge
209                                        + recheckAfter);
210                                return;
211                            }
212                        }
213                    }
214                    tile.putValue("etag", urlConn.getHeaderField("ETag"));
215                }
216                if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {
217                    // If we are isModifiedSince or If-None-Match has been set
218                    // and the server answers with a HTTP 304 = "Not Modified"
219                    log.finest("ETag test: local version is up to date: " + tile);
220                    tile.setLoaded(true);
221                    tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
222                    return;
223                }
224
225                loadTileMetadata(tile, urlConn);
226                saveTagsToFile();
227
228                if ("no-tile".equals(tile.getValue("tile-info")))
229                {
230                    tile.setError("No tile at this zoom level");
231                    listener.tileLoadingFinished(tile, true);
232                } else {
233                    for(int i = 0; i < 5; ++i) {
234                        if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {
235                            Thread.sleep(5000+(new Random()).nextInt(5000));
236                            continue;
237                        }
238                        byte[] buffer = loadTileInBuffer(urlConn);
239                        if (buffer != null) {
240                            tile.loadImage(new ByteArrayInputStream(buffer));
241                            tile.setLoaded(true);
242                            listener.tileLoadingFinished(tile, true);
243                            saveTileToFile(buffer);
244                            break;
245                        }
246                    }
247                }
248            } catch (Exception e) {
249                tile.setError(e.getMessage());
250                listener.tileLoadingFinished(tile, false);
251                if (input == null) {
252                    try {
253                        System.err.println("Failed loading " + tile.getUrl() +": " + e.getMessage());
254                    } catch(IOException i) {
255                    }
256                }
257            } finally {
258                tile.loading = false;
259                tile.setLoaded(true);
260            }
261        }
262
263        protected boolean loadTileFromFile() {
264            FileInputStream fin = null;
265            try {
266                tileFile = getTileFile();
267                if (!tileFile.exists())
268                    return false;
269
270                loadTagsFromFile();
271                if ("no-tile".equals(tile.getValue("tile-info")))
272                {
273                    tile.setError("No tile at this zoom level");
274                    if (tileFile.exists()) {
275                        tileFile.delete();
276                    }
277                    tileFile = getTagsFile();
278                } else {
279                    fin = new FileInputStream(tileFile);
280                    if (fin.available() == 0)
281                        throw new IOException("File empty");
282                    tile.loadImage(fin);
283                    fin.close();
284                }
285
286                fileAge = tileFile.lastModified();
287                boolean oldTile = System.currentTimeMillis() - fileAge > maxCacheFileAge;
288                if (!oldTile) {
289                    tile.setLoaded(true);
290                    listener.tileLoadingFinished(tile, true);
291                    fileTilePainted = true;
292                    return true;
293                }
294                listener.tileLoadingFinished(tile, true);
295                fileTilePainted = true;
296            } catch (Exception e) {
297                try {
298                    if (fin != null) {
299                        fin.close();
300                        tileFile.delete();
301                    }
302                } catch (Exception e1) {
303                }
304                tileFile = null;
305                fileAge = 0;
306            }
307            return false;
308        }
309
310        protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException {
311            input = urlConn.getInputStream();
312            try {
313                ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());
314                byte[] buffer = new byte[2048];
315                boolean finished = false;
316                do {
317                    int read = input.read(buffer);
318                    if (read >= 0) {
319                        bout.write(buffer, 0, read);
320                    } else {
321                        finished = true;
322                    }
323                } while (!finished);
324                if (bout.size() == 0)
325                    return null;
326                return bout.toByteArray();
327            } finally {
328                input.close();
329                input = null;
330            }
331        }
332
333        /**
334         * Performs a <code>HEAD</code> request for retrieving the
335         * <code>LastModified</code> header value.
336         *
337         * Note: This does only work with servers providing the
338         * <code>LastModified</code> header:
339         * <ul>
340         * <li>{@link tilesources.OsmTileSource.CycleMap} - supported</li>
341         * <li>{@link tilesources.OsmTileSource.Mapnik} - not supported</li>
342         * </ul>
343         *
344         * @param fileAge time of the
345         * @return <code>true</code> if the tile on the server is newer than the
346         *         file
347         * @throws IOException
348         */
349        protected boolean isOsmTileNewer(long fileAge) throws IOException {
350            URL url;
351            url = new URL(tile.getUrl());
352            HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
353            prepareHttpUrlConnection(urlConn);
354            urlConn.setRequestMethod("HEAD");
355            urlConn.setReadTimeout(30000); // 30 seconds read timeout
356            // System.out.println("Tile age: " + new
357            // Date(urlConn.getLastModified()) + " / "
358            // + new Date(fileAge));
359            long lastModified = urlConn.getLastModified();
360            if (lastModified == 0)
361                return true; // no LastModified time returned
362            return (lastModified > fileAge);
363        }
364
365        protected boolean hasOsmTileETag(String eTag) throws IOException {
366            URL url;
367            url = new URL(tile.getUrl());
368            HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
369            prepareHttpUrlConnection(urlConn);
370            urlConn.setRequestMethod("HEAD");
371            urlConn.setReadTimeout(30000); // 30 seconds read timeout
372            // System.out.println("Tile age: " + new
373            // Date(urlConn.getLastModified()) + " / "
374            // + new Date(fileAge));
375            String osmETag = urlConn.getHeaderField("ETag");
376            if (osmETag == null)
377                return true;
378            return (osmETag.equals(eTag));
379        }
380
381        protected File getTileFile() {
382            return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."
383                    + tile.getSource().getTileType());
384        }
385
386        protected File getTagsFile() {
387            return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile()
388                    + TAGS_FILE_EXT);
389        }
390
391        protected void saveTileToFile(byte[] rawData) {
392            try {
393                FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile()
394                        + "_" + tile.getYtile() + "." + tile.getSource().getTileType());
395                f.write(rawData);
396                f.close();
397                // System.out.println("Saved tile to file: " + tile);
398            } catch (Exception e) {
399                System.err.println("Failed to save tile content: " + e.getLocalizedMessage());
400            }
401        }
402
403        protected void saveTagsToFile() {
404            File tagsFile = getTagsFile();
405            if (tile.getMetadata() == null) {
406                tagsFile.delete();
407                return;
408            }
409            try {
410                final PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile),
411                        TAGS_CHARSET));
412                for (Entry<String, String> entry : tile.getMetadata().entrySet()) {
413                    f.println(entry.getKey() + "=" + entry.getValue());
414                }
415                f.close();
416            } catch (Exception e) {
417                System.err.println("Failed to save tile tags: " + e.getLocalizedMessage());
418            }
419        }
420
421        /** Load backward-compatiblity .etag file and if it exists move it to new .tags file*/
422        private void loadOldETagfromFile() {
423            File etagFile = new File(tileCacheDir, tile.getZoom() + "_"
424                    + tile.getXtile() + "_" + tile.getYtile() + ETAG_FILE_EXT);
425            if (!etagFile.exists()) return;
426            try {
427                FileInputStream f = new FileInputStream(etagFile);
428                byte[] buf = new byte[f.available()];
429                f.read(buf);
430                f.close();
431                String etag = new String(buf, TAGS_CHARSET.name());
432                tile.putValue("etag", etag);
433                if (etagFile.delete()) {
434                    saveTagsToFile();
435                }
436            } catch (IOException e) {
437                System.err.println("Failed to load compatiblity etag: " + e.getLocalizedMessage());
438            }
439        }
440
441        protected void loadTagsFromFile() {
442            loadOldETagfromFile();
443            File tagsFile = getTagsFile();
444            try {
445                final BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile),
446                        TAGS_CHARSET));
447                for (String line = f.readLine(); line != null; line = f.readLine()) {
448                    final int i = line.indexOf('=');
449                    if (i == -1 || i == 0) {
450                        System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line);
451                        continue;
452                    }
453                    tile.putValue(line.substring(0,i),line.substring(i+1));
454                }
455                f.close();
456            } catch (FileNotFoundException e) {
457            } catch (Exception e) {
458                System.err.println("Failed to load tile tags: " + e.getLocalizedMessage());
459            }
460        }
461
462    }
463
464    public long getMaxFileAge() {
465        return maxCacheFileAge;
466    }
467
468    /**
469     * Sets the maximum age of the local cached tile in the file system. If a
470     * local tile is older than the specified file age
471     * {@link OsmFileCacheTileLoader} will connect to the tile server and check
472     * if a newer tile is available using the mechanism specified for the
473     * selected tile source/server.
474     *
475     * @param maxFileAge
476     *            maximum age in milliseconds
477     * @see #FILE_AGE_ONE_DAY
478     * @see #FILE_AGE_ONE_WEEK
479     * @see TileSource#getTileUpdate()
480     */
481    public void setCacheMaxFileAge(long maxFileAge) {
482        this.maxCacheFileAge = maxFileAge;
483    }
484
485    public String getCacheDirBase() {
486        return cacheDirBase;
487    }
488
489    public void setTileCacheDir(String tileCacheDir) {
490        File dir = new File(tileCacheDir);
491        dir.mkdirs();
492        this.cacheDirBase = dir.getAbsolutePath();
493    }
494
495    @Override
496    public void clearCache(TileSource source) {
497        clearCache(source, null);
498    }
499
500    @Override
501    public void clearCache(TileSource source, TileClearController controller) {
502        File dir = getSourceCacheDir(source);
503        if (dir != null) {
504            if (controller != null) controller.initClearDir(dir);
505            if (dir.isDirectory()) {
506                File[] files = dir.listFiles();
507                if (controller != null) controller.initClearFiles(files);
508                for (File file : files) {
509                    if (controller != null && controller.cancel()) return;
510                    file.delete();
511                    if (controller != null) controller.fileDeleted(file);
512                }
513            }
514            dir.delete();
515        }
516        if (controller != null) controller.clearFinished();
517    }
518}