001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedInputStream;
007import java.io.BufferedOutputStream;
008import java.io.File;
009import java.io.FileInputStream;
010import java.io.FileOutputStream;
011import java.io.IOException;
012import java.io.InputStream;
013import java.net.HttpURLConnection;
014import java.net.MalformedURLException;
015import java.net.URL;
016import java.util.ArrayList;
017import java.util.Arrays;
018import java.util.Enumeration;
019import java.util.List;
020import java.util.zip.ZipEntry;
021import java.util.zip.ZipFile;
022
023import org.openstreetmap.josm.Main;
024import org.openstreetmap.josm.tools.Pair;
025import org.openstreetmap.josm.tools.Utils;
026
027/**
028 * Mirrors a file to a local file.
029 * <p>
030 * The file mirrored is only downloaded if it has been more than 7 days since last download
031 */
032public class MirroredInputStream extends InputStream {
033    InputStream fs = null;
034    File file = null;
035
036    public final static long DEFAULT_MAXTIME = -1L;
037
038    public MirroredInputStream(String name) throws IOException {
039        this(name, null, DEFAULT_MAXTIME);
040    }
041
042    public MirroredInputStream(String name, long maxTime) throws IOException {
043        this(name, null, maxTime);
044    }
045
046    public MirroredInputStream(String name, String destDir) throws IOException {
047        this(name, destDir, DEFAULT_MAXTIME);
048    }
049
050    /**
051     * Get an inputstream from a given filename, url or internal resource.
052     * @param name can be
053     *  - relative or absolute file name
054     *  - file:///SOME/FILE the same as above
055     *  - resource://SOME/FILE file from the classpath (usually in the current *.jar)
056     *  - http://... a url. It will be cached on disk.
057     * @param destDir the destination directory for the cache file. only applies for urls.
058     * @param maxTime the maximum age of the cache file (in seconds)
059     * @throws IOException when the resource with the given name could not be retrieved
060     */
061    public MirroredInputStream(String name, String destDir, long maxTime) throws IOException {
062        URL url;
063        try {
064            url = new URL(name);
065            if (url.getProtocol().equals("file")) {
066                file = new File(name.substring("file:/".length()));
067                if (!file.exists()) {
068                    file = new File(name.substring("file://".length()));
069                }
070            } else {
071                if (Main.applet) {
072                    fs = new BufferedInputStream(Utils.openURL(url));
073                    file = new File(url.getFile());
074                } else {
075                    file = checkLocal(url, destDir, maxTime);
076                }
077            }
078        } catch (java.net.MalformedURLException e) {
079            if (name.startsWith("resource://")) {
080                fs = getClass().getResourceAsStream(
081                        name.substring("resource:/".length()));
082                if (fs == null)
083                    throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
084                return;
085            }
086            file = new File(name);
087        }
088        if (file == null)
089            throw new IOException();
090        fs = new FileInputStream(file);
091    }
092
093    /**
094     * Looks for a certain entry inside a zip file and returns the entry path.
095     *
096     * Replies a file in the top level directory of the ZIP file which has an
097     * extension <code>extension</code>. If more than one files have this
098     * extension, the last file whose name includes <code>namepart</code>
099     * is opened.
100     *
101     * @param extension  the extension of the file we're looking for
102     * @param namepart the name part
103     * @return The zip entry path of the matching file. Null if this mirrored
104     * input stream doesn't represent a zip file or if there was no matching
105     * file in the ZIP file.
106     */
107    public String findZipEntryPath(String extension, String namepart) {
108        Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
109        if (ze == null) return null;
110        return ze.a;
111    }
112
113    /**
114     * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
115     */
116    public InputStream findZipEntryInputStream(String extension, String namepart) {
117        Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
118        if (ze == null) return null;
119        return ze.b;
120    }
121
122    @Deprecated // use findZipEntryInputStream
123    public InputStream getZipEntry(String extension, String namepart) {
124        return findZipEntryInputStream(extension, namepart);
125    }
126
127    private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
128        if (file == null)
129            return null;
130        Pair<String, InputStream> res = null;
131        try {
132            ZipFile zipFile = new ZipFile(file);
133            ZipEntry resentry = null;
134            Enumeration<? extends ZipEntry> entries = zipFile.entries();
135            while (entries.hasMoreElements()) {
136                ZipEntry entry = entries.nextElement();
137                if (entry.getName().endsWith("." + extension)) {
138                    /* choose any file with correct extension. When more than
139                        one file, prefer the one which matches namepart */
140                    if (resentry == null || entry.getName().indexOf(namepart) >= 0) {
141                        resentry = entry;
142                    }
143                }
144            }
145            if (resentry != null) {
146                InputStream is = zipFile.getInputStream(resentry);
147                res = Pair.create(resentry.getName(), is);
148            } else {
149                Utils.close(zipFile);
150            }
151        } catch (Exception e) {
152            if (file.getName().endsWith(".zip")) {
153                Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
154                        file.getName(), e.toString(), extension, namepart));
155            }
156        }
157        return res;
158    }
159
160    public File getFile() {
161        return file;
162    }
163
164    public static void cleanup(String name) {
165        cleanup(name, null);
166    }
167    
168    public static void cleanup(String name, String destDir) {
169        URL url;
170        try {
171            url = new URL(name);
172            if (!url.getProtocol().equals("file")) {
173                String prefKey = getPrefKey(url, destDir);
174                List<String> localPath = new ArrayList<String>(Main.pref.getCollection(prefKey));
175                if (localPath.size() == 2) {
176                    File lfile = new File(localPath.get(1));
177                    if(lfile.exists()) {
178                        lfile.delete();
179                    }
180                }
181                Main.pref.putCollection(prefKey, null);
182            }
183        } catch (MalformedURLException e) {
184            Main.warn(e);
185        }
186    }
187
188    /**
189     * get preference key to store the location and age of the cached file.
190     * 2 resources that point to the same url, but that are to be stored in different
191     * directories will not share a cache file.
192     */
193    private static String getPrefKey(URL url, String destDir) {
194        StringBuilder prefKey = new StringBuilder("mirror.");
195        if (destDir != null) {
196            prefKey.append(destDir);
197            prefKey.append(".");
198        }
199        prefKey.append(url.toString());
200        return prefKey.toString().replaceAll("=","_");
201    }
202
203    private File checkLocal(URL url, String destDir, long maxTime) throws IOException {
204        String prefKey = getPrefKey(url, destDir);
205        long age = 0L;
206        File localFile = null;
207        List<String> localPathEntry = new ArrayList<String>(Main.pref.getCollection(prefKey));
208        if (localPathEntry.size() == 2) {
209            localFile = new File(localPathEntry.get(1));
210            if(!localFile.exists())
211                localFile = null;
212            else {
213                if ( maxTime == DEFAULT_MAXTIME
214                        || maxTime <= 0 // arbitrary value <= 0 is deprecated
215                ) {
216                    maxTime = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week
217                }
218                age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
219                if (age < maxTime*1000) {
220                    return localFile;
221                }
222            }
223        }
224        if (destDir == null) {
225            destDir = Main.pref.getCacheDirectory().getPath();
226        }
227
228        File destDirFile = new File(destDir);
229        if (!destDirFile.exists()) {
230            destDirFile.mkdirs();
231        }
232
233        String a = url.toString().replaceAll("[^A-Za-z0-9_.-]", "_");
234        String localPath = "mirror_" + a;
235        destDirFile = new File(destDir, localPath + ".tmp");
236        BufferedOutputStream bos = null;
237        BufferedInputStream bis = null;
238        try {
239            HttpURLConnection con = connectFollowingRedirect(url);
240            bis = new BufferedInputStream(con.getInputStream());
241            FileOutputStream fos = new FileOutputStream(destDirFile);
242            bos = new BufferedOutputStream(fos);
243            byte[] buffer = new byte[4096];
244            int length;
245            while ((length = bis.read(buffer)) > -1) {
246                bos.write(buffer, 0, length);
247            }
248            Utils.close(bos);
249            bos = null;
250            /* close fos as well to be sure! */
251            Utils.close(fos);
252            fos = null;
253            localFile = new File(destDir, localPath);
254            if(Main.platform.rename(destDirFile, localFile)) {
255                Main.pref.putCollection(prefKey, Arrays.asList(new String[]
256                {Long.toString(System.currentTimeMillis()), localFile.toString()}));
257            } else {
258                Main.warn(tr("Failed to rename file {0} to {1}.",
259                destDirFile.getPath(), localFile.getPath()));
260            }
261        } catch (IOException e) {
262            if (age >= maxTime*1000 && age < maxTime*1000*2) {
263                Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", url, e));
264                return localFile;
265            } else {
266                throw e;
267            }
268        } finally {
269            Utils.close(bis);
270            Utils.close(bos);
271        }
272
273        return localFile;
274    }
275
276    /**
277     * Opens a connection for downloading a resource.
278     * <p>
279     * Manually follows redirects because
280     * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect
281     * is going from a http to a https URL, see <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4620571">bug report</a>.
282     * <p>
283     * This can causes problems when downloading from certain GitHub URLs.
284     * 
285     * @param downloadUrl The resource URL to download
286     * @return The HTTP connection effectively linked to the resource, after all potential redirections
287     * @throws MalformedURLException If a redirected URL is wrong
288     * @throws IOException If any I/O operation goes wrong
289     * @since 6073
290     */
291    public static HttpURLConnection connectFollowingRedirect(URL downloadUrl) throws MalformedURLException, IOException {
292        HttpURLConnection con = null;
293        int numRedirects = 0;
294        while(true) {
295            con = Utils.openHttpConnection(downloadUrl);
296            con.setInstanceFollowRedirects(false);
297            con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000);
298            con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000);
299            con.connect();
300            switch(con.getResponseCode()) {
301            case HttpURLConnection.HTTP_OK:
302                return con;
303            case HttpURLConnection.HTTP_MOVED_PERM:
304            case HttpURLConnection.HTTP_MOVED_TEMP:
305            case HttpURLConnection.HTTP_SEE_OTHER:
306                String redirectLocation = con.getHeaderField("Location");
307                if (downloadUrl == null) {
308                    /* I18n: argument is HTTP response code */ String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header. Can''t redirect. Aborting.", con.getResponseCode());
309                    throw new IOException(msg);
310                }
311                downloadUrl = new URL(redirectLocation);
312                // keep track of redirect attempts to break a redirect loops if it happens
313                // to occur for whatever reason
314                numRedirects++;
315                if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) {
316                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
317                    throw new IOException(msg);
318                }
319                Main.info(tr("Download redirected to ''{0}''", downloadUrl));
320                break;
321            default:
322                String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadUrl, con.getResponseCode());
323                throw new IOException(msg);
324            }
325        }
326    }
327
328    @Override
329    public int available() throws IOException
330    { return fs.available(); }
331    @Override
332    public void close() throws IOException
333    { Utils.close(fs); }
334    @Override
335    public int read() throws IOException
336    { return fs.read(); }
337    @Override
338    public int read(byte[] b) throws IOException
339    { return fs.read(b); }
340    @Override
341    public int read(byte[] b, int off, int len) throws IOException
342    { return fs.read(b,off, len); }
343    @Override
344    public long skip(long n) throws IOException
345    { return fs.skip(n); }
346}