001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004 005import java.awt.Graphics2D; 006import java.awt.image.BufferedImage; 007import java.io.BufferedOutputStream; 008import java.io.File; 009import java.io.FileInputStream; 010import java.io.FileNotFoundException; 011import java.io.FileOutputStream; 012import java.io.IOException; 013import java.io.InputStream; 014import java.io.OutputStream; 015import java.lang.ref.SoftReference; 016import java.net.URLConnection; 017import java.text.SimpleDateFormat; 018import java.util.ArrayList; 019import java.util.Calendar; 020import java.util.Collections; 021import java.util.Comparator; 022import java.util.HashMap; 023import java.util.HashSet; 024import java.util.Iterator; 025import java.util.List; 026import java.util.Map; 027import java.util.Properties; 028import java.util.Set; 029 030import javax.imageio.ImageIO; 031import javax.xml.bind.JAXBContext; 032import javax.xml.bind.Marshaller; 033import javax.xml.bind.Unmarshaller; 034 035import org.openstreetmap.josm.Main; 036import org.openstreetmap.josm.data.ProjectionBounds; 037import org.openstreetmap.josm.data.coor.EastNorth; 038import org.openstreetmap.josm.data.coor.LatLon; 039import org.openstreetmap.josm.data.imagery.types.EntryType; 040import org.openstreetmap.josm.data.imagery.types.ProjectionType; 041import org.openstreetmap.josm.data.imagery.types.WmsCacheType; 042import org.openstreetmap.josm.data.preferences.StringProperty; 043import org.openstreetmap.josm.data.projection.Projection; 044import org.openstreetmap.josm.gui.NavigatableComponent; 045import org.openstreetmap.josm.tools.Utils; 046 047 048 049public class WmsCache { 050 //TODO Property for maximum cache size 051 //TODO Property for maximum age of tile, automatically remove old tiles 052 //TODO Measure time for partially loading from cache, compare with time to download tile. If slower, disable partial cache 053 //TODO Do loading from partial cache and downloading at the same time, don't wait for partical cache to load 054 055 private static final StringProperty PROP_CACHE_PATH = new StringProperty("imagery.wms-cache.path", "wms"); 056 private static final String INDEX_FILENAME = "index.xml"; 057 private static final String LAYERS_INDEX_FILENAME = "layers.properties"; 058 059 private static class CacheEntry { 060 final double pixelPerDegree; 061 final double east; 062 final double north; 063 final ProjectionBounds bounds; 064 final String filename; 065 066 long lastUsed; 067 long lastModified; 068 069 CacheEntry(double pixelPerDegree, double east, double north, int tileSize, String filename) { 070 this.pixelPerDegree = pixelPerDegree; 071 this.east = east; 072 this.north = north; 073 this.bounds = new ProjectionBounds(east, north, east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree); 074 this.filename = filename; 075 } 076 } 077 078 private static class ProjectionEntries { 079 final String projection; 080 final String cacheDirectory; 081 final List<CacheEntry> entries = new ArrayList<WmsCache.CacheEntry>(); 082 083 ProjectionEntries(String projection, String cacheDirectory) { 084 this.projection = projection; 085 this.cacheDirectory = cacheDirectory; 086 } 087 } 088 089 private final Map<String, ProjectionEntries> entries = new HashMap<String, ProjectionEntries>(); 090 private final File cacheDir; 091 private final int tileSize; // Should be always 500 092 private int totalFileSize; 093 private boolean totalFileSizeDirty; // Some file was missing - size needs to be recalculated 094 // No need for hashCode/equals on CacheEntry, object identity is enough. Comparing by values can lead to error - CacheEntry for wrong projection could be found 095 private Map<CacheEntry, SoftReference<BufferedImage>> memoryCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>(); 096 private Set<ProjectionBounds> areaToCache; 097 098 protected String cacheDirPath() { 099 String cPath = PROP_CACHE_PATH.get(); 100 if (!(new File(cPath).isAbsolute())) { 101 cPath = Main.pref.getCacheDirectory() + File.separator + cPath; 102 } 103 return cPath; 104 } 105 106 public WmsCache(String url, int tileSize) { 107 File globalCacheDir = new File(cacheDirPath()); 108 globalCacheDir.mkdirs(); 109 cacheDir = new File(globalCacheDir, getCacheDirectory(url)); 110 cacheDir.mkdirs(); 111 this.tileSize = tileSize; 112 } 113 114 private String getCacheDirectory(String url) { 115 String cacheDirName = null; 116 InputStream fis = null; 117 OutputStream fos = null; 118 try { 119 Properties layersIndex = new Properties(); 120 File layerIndexFile = new File(cacheDirPath(), LAYERS_INDEX_FILENAME); 121 try { 122 fis = new FileInputStream(layerIndexFile); 123 layersIndex.load(fis); 124 } catch (FileNotFoundException e) { 125 Main.error("Unable to load layers index for wms cache (file " + layerIndexFile + " not found)"); 126 } catch (IOException e) { 127 Main.error("Unable to load layers index for wms cache"); 128 e.printStackTrace(); 129 } 130 131 for (Object propKey: layersIndex.keySet()) { 132 String s = (String)propKey; 133 if (url.equals(layersIndex.getProperty(s))) { 134 cacheDirName = s; 135 break; 136 } 137 } 138 139 if (cacheDirName == null) { 140 int counter = 0; 141 while (true) { 142 counter++; 143 if (!layersIndex.keySet().contains(String.valueOf(counter))) { 144 break; 145 } 146 } 147 cacheDirName = String.valueOf(counter); 148 layersIndex.setProperty(cacheDirName, url); 149 try { 150 fos = new FileOutputStream(layerIndexFile); 151 layersIndex.store(fos, ""); 152 } catch (IOException e) { 153 Main.error("Unable to save layer index for wms cache"); 154 e.printStackTrace(); 155 } 156 } 157 } finally { 158 Utils.close(fos); 159 Utils.close(fis); 160 } 161 162 return cacheDirName; 163 } 164 165 private ProjectionEntries getProjectionEntries(Projection projection) { 166 return getProjectionEntries(projection.toCode(), projection.getCacheDirectoryName()); 167 } 168 169 private ProjectionEntries getProjectionEntries(String projection, String cacheDirectory) { 170 ProjectionEntries result = entries.get(projection); 171 if (result == null) { 172 result = new ProjectionEntries(projection, cacheDirectory); 173 entries.put(projection, result); 174 } 175 176 return result; 177 } 178 179 public synchronized void loadIndex() { 180 File indexFile = new File(cacheDir, INDEX_FILENAME); 181 try { 182 JAXBContext context = JAXBContext.newInstance( 183 WmsCacheType.class.getPackage().getName(), 184 WmsCacheType.class.getClassLoader()); 185 Unmarshaller unmarshaller = context.createUnmarshaller(); 186 WmsCacheType cacheEntries = (WmsCacheType)unmarshaller.unmarshal(new FileInputStream(indexFile)); 187 totalFileSize = cacheEntries.getTotalFileSize(); 188 if (cacheEntries.getTileSize() != tileSize) { 189 Main.info("Cache created with different tileSize, cache will be discarded"); 190 return; 191 } 192 for (ProjectionType projectionType: cacheEntries.getProjection()) { 193 ProjectionEntries projection = getProjectionEntries(projectionType.getName(), projectionType.getCacheDirectory()); 194 for (EntryType entry: projectionType.getEntry()) { 195 CacheEntry ce = new CacheEntry(entry.getPixelPerDegree(), entry.getEast(), entry.getNorth(), tileSize, entry.getFilename()); 196 ce.lastUsed = entry.getLastUsed().getTimeInMillis(); 197 ce.lastModified = entry.getLastModified().getTimeInMillis(); 198 projection.entries.add(ce); 199 } 200 } 201 } catch (Exception e) { 202 if (indexFile.exists()) { 203 e.printStackTrace(); 204 Main.info("Unable to load index for wms-cache, new file will be created"); 205 } else { 206 Main.info("Index for wms-cache doesn't exist, new file will be created"); 207 } 208 } 209 210 removeNonReferencedFiles(); 211 } 212 213 private void removeNonReferencedFiles() { 214 215 Set<String> usedProjections = new HashSet<String>(); 216 217 for (ProjectionEntries projectionEntries: entries.values()) { 218 219 usedProjections.add(projectionEntries.cacheDirectory); 220 221 File projectionDir = new File(cacheDir, projectionEntries.cacheDirectory); 222 if (projectionDir.exists()) { 223 Set<String> referencedFiles = new HashSet<String>(); 224 225 for (CacheEntry ce: projectionEntries.entries) { 226 referencedFiles.add(ce.filename); 227 } 228 229 for (File file: projectionDir.listFiles()) { 230 if (!referencedFiles.contains(file.getName())) { 231 file.delete(); 232 } 233 } 234 } 235 } 236 237 for (File projectionDir: cacheDir.listFiles()) { 238 if (projectionDir.isDirectory() && !usedProjections.contains(projectionDir.getName())) { 239 Utils.deleteDirectory(projectionDir); 240 } 241 } 242 } 243 244 private int calculateTotalFileSize() { 245 int result = 0; 246 for (ProjectionEntries projectionEntries: entries.values()) { 247 Iterator<CacheEntry> it = projectionEntries.entries.iterator(); 248 while (it.hasNext()) { 249 CacheEntry entry = it.next(); 250 File imageFile = getImageFile(projectionEntries, entry); 251 if (!imageFile.exists()) { 252 it.remove(); 253 } else { 254 result += imageFile.length(); 255 } 256 } 257 } 258 return result; 259 } 260 261 public synchronized void saveIndex() { 262 WmsCacheType index = new WmsCacheType(); 263 264 if (totalFileSizeDirty) { 265 totalFileSize = calculateTotalFileSize(); 266 } 267 268 index.setTileSize(tileSize); 269 index.setTotalFileSize(totalFileSize); 270 for (ProjectionEntries projectionEntries: entries.values()) { 271 if (!projectionEntries.entries.isEmpty()) { 272 ProjectionType projectionType = new ProjectionType(); 273 projectionType.setName(projectionEntries.projection); 274 projectionType.setCacheDirectory(projectionEntries.cacheDirectory); 275 index.getProjection().add(projectionType); 276 for (CacheEntry ce: projectionEntries.entries) { 277 EntryType entry = new EntryType(); 278 entry.setPixelPerDegree(ce.pixelPerDegree); 279 entry.setEast(ce.east); 280 entry.setNorth(ce.north); 281 Calendar c = Calendar.getInstance(); 282 c.setTimeInMillis(ce.lastUsed); 283 entry.setLastUsed(c); 284 c = Calendar.getInstance(); 285 c.setTimeInMillis(ce.lastModified); 286 entry.setLastModified(c); 287 entry.setFilename(ce.filename); 288 projectionType.getEntry().add(entry); 289 } 290 } 291 } 292 try { 293 JAXBContext context = JAXBContext.newInstance( 294 WmsCacheType.class.getPackage().getName(), 295 WmsCacheType.class.getClassLoader()); 296 Marshaller marshaller = context.createMarshaller(); 297 marshaller.marshal(index, new FileOutputStream(new File(cacheDir, INDEX_FILENAME))); 298 } catch (Exception e) { 299 Main.error("Failed to save wms-cache file"); 300 e.printStackTrace(); 301 } 302 } 303 304 private File getImageFile(ProjectionEntries projection, CacheEntry entry) { 305 return new File(cacheDir, projection.cacheDirectory + "/" + entry.filename); 306 } 307 308 309 private BufferedImage loadImage(ProjectionEntries projectionEntries, CacheEntry entry) throws IOException { 310 311 synchronized (this) { 312 entry.lastUsed = System.currentTimeMillis(); 313 314 SoftReference<BufferedImage> memCache = memoryCache.get(entry); 315 if (memCache != null) { 316 BufferedImage result = memCache.get(); 317 if (result != null) 318 return result; 319 } 320 } 321 322 try { 323 // Reading can't be in synchronized section, it's too slow 324 BufferedImage result = ImageIO.read(getImageFile(projectionEntries, entry)); 325 synchronized (this) { 326 if (result == null) { 327 projectionEntries.entries.remove(entry); 328 totalFileSizeDirty = true; 329 } 330 return result; 331 } 332 } catch (IOException e) { 333 synchronized (this) { 334 projectionEntries.entries.remove(entry); 335 totalFileSizeDirty = true; 336 throw e; 337 } 338 } 339 } 340 341 private CacheEntry findEntry(ProjectionEntries projectionEntries, double pixelPerDegree, double east, double north) { 342 for (CacheEntry entry: projectionEntries.entries) { 343 if (entry.pixelPerDegree == pixelPerDegree && entry.east == east && entry.north == north) 344 return entry; 345 } 346 return null; 347 } 348 349 public synchronized boolean hasExactMatch(Projection projection, double pixelPerDegree, double east, double north) { 350 ProjectionEntries projectionEntries = getProjectionEntries(projection); 351 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north); 352 return (entry != null); 353 } 354 355 public BufferedImage getExactMatch(Projection projection, double pixelPerDegree, double east, double north) { 356 CacheEntry entry = null; 357 ProjectionEntries projectionEntries = null; 358 synchronized (this) { 359 projectionEntries = getProjectionEntries(projection); 360 entry = findEntry(projectionEntries, pixelPerDegree, east, north); 361 } 362 if (entry != null) { 363 try { 364 return loadImage(projectionEntries, entry); 365 } catch (IOException e) { 366 Main.error("Unable to load file from wms cache"); 367 e.printStackTrace(); 368 return null; 369 } 370 } 371 return null; 372 } 373 374 public BufferedImage getPartialMatch(Projection projection, double pixelPerDegree, double east, double north) { 375 ProjectionEntries projectionEntries; 376 List<CacheEntry> matches; 377 synchronized (this) { 378 matches = new ArrayList<WmsCache.CacheEntry>(); 379 380 double minPPD = pixelPerDegree / 5; 381 double maxPPD = pixelPerDegree * 5; 382 projectionEntries = getProjectionEntries(projection); 383 384 double size2 = tileSize / pixelPerDegree; 385 double border = tileSize * 0.01; // Make sure not to load neighboring tiles that intersects this tile only slightly 386 ProjectionBounds bounds = new ProjectionBounds(east + border, north + border, 387 east + size2 - border, north + size2 - border); 388 389 //TODO Do not load tile if it is completely overlapped by other tile with better ppd 390 for (CacheEntry entry: projectionEntries.entries) { 391 if (entry.pixelPerDegree >= minPPD && entry.pixelPerDegree <= maxPPD && entry.bounds.intersects(bounds)) { 392 entry.lastUsed = System.currentTimeMillis(); 393 matches.add(entry); 394 } 395 } 396 397 if (matches.isEmpty()) 398 return null; 399 400 401 Collections.sort(matches, new Comparator<CacheEntry>() { 402 @Override 403 public int compare(CacheEntry o1, CacheEntry o2) { 404 return Double.compare(o2.pixelPerDegree, o1.pixelPerDegree); 405 } 406 }); 407 } 408 409 //TODO Use alpha layer only when enabled on wms layer 410 BufferedImage result = new BufferedImage(tileSize, tileSize, BufferedImage.TYPE_4BYTE_ABGR); 411 Graphics2D g = result.createGraphics(); 412 413 414 boolean drawAtLeastOnce = false; 415 Map<CacheEntry, SoftReference<BufferedImage>> localCache = new HashMap<WmsCache.CacheEntry, SoftReference<BufferedImage>>(); 416 for (CacheEntry ce: matches) { 417 BufferedImage img; 418 try { 419 img = loadImage(projectionEntries, ce); 420 localCache.put(ce, new SoftReference<BufferedImage>(img)); 421 } catch (IOException e) { 422 continue; 423 } 424 425 drawAtLeastOnce = true; 426 427 int xDiff = (int)((ce.east - east) * pixelPerDegree); 428 int yDiff = (int)((ce.north - north) * pixelPerDegree); 429 int size = (int)(pixelPerDegree / ce.pixelPerDegree * tileSize); 430 431 int x = xDiff; 432 int y = -size + tileSize - yDiff; 433 434 g.drawImage(img, x, y, size, size, null); 435 } 436 437 if (drawAtLeastOnce) { 438 synchronized (this) { 439 memoryCache.putAll(localCache); 440 } 441 return result; 442 } else 443 return null; 444 } 445 446 private String generateFileName(ProjectionEntries projectionEntries, double pixelPerDegree, Projection projection, double east, double north, String mimeType) { 447 LatLon ll1 = projection.eastNorth2latlon(new EastNorth(east, north)); 448 LatLon ll2 = projection.eastNorth2latlon(new EastNorth(east + 100 / pixelPerDegree, north)); 449 LatLon ll3 = projection.eastNorth2latlon(new EastNorth(east + tileSize / pixelPerDegree, north + tileSize / pixelPerDegree)); 450 451 double deltaLat = Math.abs(ll3.lat() - ll1.lat()); 452 double deltaLon = Math.abs(ll3.lon() - ll1.lon()); 453 int precisionLat = Math.max(0, -(int)Math.ceil(Math.log10(deltaLat)) + 1); 454 int precisionLon = Math.max(0, -(int)Math.ceil(Math.log10(deltaLon)) + 1); 455 456 String zoom = NavigatableComponent.METRIC_SOM.getDistText(ll1.greatCircleDistance(ll2)); 457 String extension; 458 if ("image/jpeg".equals(mimeType) || "image/jpg".equals(mimeType)) { 459 extension = "jpg"; 460 } else if ("image/png".equals(mimeType)) { 461 extension = "png"; 462 } else if ("image/gif".equals(mimeType)) { 463 extension = "gif"; 464 } else { 465 extension = "dat"; 466 } 467 468 int counter = 0; 469 FILENAME_LOOP: 470 while (true) { 471 String result = String.format("%s_%." + precisionLat + "f_%." + precisionLon +"f%s.%s", zoom, ll1.lat(), ll1.lon(), counter==0?"":"_" + counter, extension); 472 for (CacheEntry entry: projectionEntries.entries) { 473 if (entry.filename.equals(result)) { 474 counter++; 475 continue FILENAME_LOOP; 476 } 477 } 478 return result; 479 } 480 } 481 482 /** 483 * 484 * @param img Used only when overlapping is used, when not used, used raw from imageData 485 * @param imageData 486 * @param projection 487 * @param pixelPerDegree 488 * @param east 489 * @param north 490 * @throws IOException 491 */ 492 public synchronized void saveToCache(BufferedImage img, InputStream imageData, Projection projection, double pixelPerDegree, double east, double north) throws IOException { 493 ProjectionEntries projectionEntries = getProjectionEntries(projection); 494 CacheEntry entry = findEntry(projectionEntries, pixelPerDegree, east, north); 495 File imageFile; 496 if (entry == null) { 497 498 String mimeType; 499 if (img != null) { 500 mimeType = "image/png"; 501 } else { 502 mimeType = URLConnection.guessContentTypeFromStream(imageData); 503 } 504 entry = new CacheEntry(pixelPerDegree, east, north, tileSize,generateFileName(projectionEntries, pixelPerDegree, projection, east, north, mimeType)); 505 entry.lastUsed = System.currentTimeMillis(); 506 entry.lastModified = entry.lastUsed; 507 projectionEntries.entries.add(entry); 508 imageFile = getImageFile(projectionEntries, entry); 509 } else { 510 imageFile = getImageFile(projectionEntries, entry); 511 totalFileSize -= imageFile.length(); 512 } 513 514 imageFile.getParentFile().mkdirs(); 515 516 if (img != null) { 517 BufferedImage copy = new BufferedImage(tileSize, tileSize, img.getType()); 518 copy.createGraphics().drawImage(img, 0, 0, tileSize, tileSize, 0, img.getHeight() - tileSize, tileSize, img.getHeight(), null); 519 ImageIO.write(copy, "png", imageFile); 520 totalFileSize += imageFile.length(); 521 } else { 522 OutputStream os = new BufferedOutputStream(new FileOutputStream(imageFile)); 523 try { 524 totalFileSize += Utils.copyStream(imageData, os); 525 } finally { 526 Utils.close(os); 527 } 528 } 529 } 530 531 public synchronized void cleanSmallFiles(int size) { 532 for (ProjectionEntries projectionEntries: entries.values()) { 533 Iterator<CacheEntry> it = projectionEntries.entries.iterator(); 534 while (it.hasNext()) { 535 File file = getImageFile(projectionEntries, it.next()); 536 long length = file.length(); 537 if (length <= size) { 538 if (length == 0) { 539 totalFileSizeDirty = true; // File probably doesn't exist 540 } 541 totalFileSize -= size; 542 file.delete(); 543 it.remove(); 544 } 545 } 546 } 547 } 548 549 public static String printDate(Calendar c) { 550 return (new SimpleDateFormat("yyyy-MM-dd")).format(c.getTime()); 551 } 552 553 private boolean isInsideAreaToCache(CacheEntry cacheEntry) { 554 for (ProjectionBounds b: areaToCache) { 555 if (cacheEntry.bounds.intersects(b)) 556 return true; 557 } 558 return false; 559 } 560 561 public synchronized void setAreaToCache(Set<ProjectionBounds> areaToCache) { 562 this.areaToCache = areaToCache; 563 Iterator<CacheEntry> it = memoryCache.keySet().iterator(); 564 while (it.hasNext()) { 565 if (!isInsideAreaToCache(it.next())) { 566 it.remove(); 567 } 568 } 569 } 570}