001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.tools;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedOutputStream;
007import java.io.BufferedReader;
008import java.io.IOException;
009import java.io.InputStream;
010import java.io.OutputStream;
011import java.net.HttpRetryException;
012import java.net.HttpURLConnection;
013import java.net.URL;
014import java.util.List;
015import java.util.Locale;
016import java.util.Map;
017import java.util.Scanner;
018import java.util.TreeMap;
019import java.util.regex.Matcher;
020import java.util.regex.Pattern;
021import java.util.zip.GZIPInputStream;
022
023import org.openstreetmap.josm.Main;
024import org.openstreetmap.josm.data.Version;
025import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
026import org.openstreetmap.josm.gui.progress.ProgressMonitor;
027import org.openstreetmap.josm.io.Compression;
028import org.openstreetmap.josm.io.ProgressInputStream;
029import org.openstreetmap.josm.io.ProgressOutputStream;
030import org.openstreetmap.josm.io.UTFInputStreamReader;
031
032/**
033 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}.
034 * @since 9168
035 */
036public final class HttpClient {
037
038    private URL url;
039    private final String requestMethod;
040    private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000;
041    private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000;
042    private byte[] requestBody;
043    private long ifModifiedSince;
044    private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
045    private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5);
046    private boolean useCache;
047    private String reasonForRequest;
048    private transient HttpURLConnection connection; // to allow disconnecting before `response` is set
049    private transient Response response;
050
051    private HttpClient(URL url, String requestMethod) {
052        this.url = url;
053        this.requestMethod = requestMethod;
054        this.headers.put("Accept-Encoding", "gzip");
055    }
056
057    /**
058     * Opens the HTTP connection.
059     * @return HTTP response
060     * @throws IOException if any I/O error occurs
061     */
062    public Response connect() throws IOException {
063        return connect(null);
064    }
065
066    /**
067     * Opens the HTTP connection.
068     * @param progressMonitor progress monitor
069     * @return HTTP response
070     * @throws IOException if any I/O error occurs
071     * @since 9179
072     */
073    public Response connect(ProgressMonitor progressMonitor) throws IOException {
074        if (progressMonitor == null) {
075            progressMonitor = NullProgressMonitor.INSTANCE;
076        }
077        final HttpURLConnection connection = (HttpURLConnection) url.openConnection();
078        this.connection = connection;
079        connection.setRequestMethod(requestMethod);
080        connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString());
081        connection.setConnectTimeout(connectTimeout);
082        connection.setReadTimeout(readTimeout);
083        connection.setInstanceFollowRedirects(false); // we do that ourselves
084        if (ifModifiedSince > 0) {
085            connection.setIfModifiedSince(ifModifiedSince);
086        }
087        connection.setUseCaches(useCache);
088        if (!useCache) {
089            connection.setRequestProperty("Cache-Control", "no-cache");
090        }
091        for (Map.Entry<String, String> header : headers.entrySet()) {
092            if (header.getValue() != null) {
093                connection.setRequestProperty(header.getKey(), header.getValue());
094            }
095        }
096
097        progressMonitor.beginTask(tr("Contacting Server..."), 1);
098        progressMonitor.indeterminateSubTask(null);
099
100        if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
101            Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault()));
102            connection.setFixedLengthStreamingMode(requestBody.length);
103            connection.setDoOutput(true);
104            try (OutputStream out = new BufferedOutputStream(
105                    new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor))) {
106                out.write(requestBody);
107            }
108        }
109
110        boolean successfulConnection = false;
111        try {
112            try {
113                connection.connect();
114                final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty();
115                Main.info("{0} {1}{2} -> {3}{4}",
116                        requestMethod, url, hasReason ? " (" + reasonForRequest + ")" : "",
117                        connection.getResponseCode(),
118                        connection.getContentLengthLong() > 0
119                                ? " (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ")"
120                                : ""
121                );
122                if (Main.isDebugEnabled()) {
123                    Main.debug("RESPONSE: " + connection.getHeaderFields());
124                }
125            } catch (IOException e) {
126                Main.info("{0} {1} -> !!!", requestMethod, url);
127                Main.warn(e);
128                //noinspection ThrowableResultOfMethodCallIgnored
129                Main.addNetworkError(url, Utils.getRootCause(e));
130                throw e;
131            }
132            if (isRedirect(connection.getResponseCode())) {
133                final String redirectLocation = connection.getHeaderField("Location");
134                if (redirectLocation == null) {
135                    /* I18n: argument is HTTP response code */
136                    String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." +
137                            " Can''t redirect. Aborting.", connection.getResponseCode());
138                    throw new IOException(msg);
139                } else if (maxRedirects > 0) {
140                    url = new URL(url, redirectLocation);
141                    maxRedirects--;
142                    Main.info(tr("Download redirected to ''{0}''", redirectLocation));
143                    return connect();
144                } else if (maxRedirects == 0) {
145                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
146                    throw new IOException(msg);
147                }
148            }
149            response = new Response(connection, progressMonitor);
150            successfulConnection = true;
151            return response;
152        } finally {
153            if (!successfulConnection) {
154                connection.disconnect();
155            }
156        }
157    }
158
159    /**
160     * Returns the HTTP response which is set only after calling {@link #connect()}.
161     * Calling this method again, returns the identical object (unless another {@link #connect()} is performed).
162     *
163     * @return the HTTP response
164     * @since 9309
165     */
166    public Response getResponse() {
167        return response;
168    }
169
170    /**
171     * A wrapper for the HTTP response.
172     */
173    public static final class Response {
174        private final HttpURLConnection connection;
175        private final ProgressMonitor monitor;
176        private final int responseCode;
177        private final String responseMessage;
178        private boolean uncompress;
179        private boolean uncompressAccordingToContentDisposition;
180
181        private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException {
182            CheckParameterUtil.ensureParameterNotNull(connection, "connection");
183            CheckParameterUtil.ensureParameterNotNull(monitor, "monitor");
184            this.connection = connection;
185            this.monitor = monitor;
186            this.responseCode = connection.getResponseCode();
187            this.responseMessage = connection.getResponseMessage();
188        }
189
190        /**
191         * Sets whether {@link #getContent()} should uncompress the input stream if necessary.
192         *
193         * @param uncompress whether the input stream should be uncompressed if necessary
194         * @return {@code this}
195         */
196        public Response uncompress(boolean uncompress) {
197            this.uncompress = uncompress;
198            return this;
199        }
200
201        /**
202         * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition}
203         * HTTP header.
204         * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to
205         * {@code Content-Disposition}
206         * @return {@code this}
207         * @since 9172
208         */
209        public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) {
210            this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition;
211            return this;
212        }
213
214        /**
215         * Returns the URL.
216         * @return the URL
217         * @see HttpURLConnection#getURL()
218         * @since 9172
219         */
220        public URL getURL() {
221            return connection.getURL();
222        }
223
224        /**
225         * Returns the request method.
226         * @return the HTTP request method
227         * @see HttpURLConnection#getRequestMethod()
228         * @since 9172
229         */
230        public String getRequestMethod() {
231            return connection.getRequestMethod();
232        }
233
234        /**
235         * Returns an input stream that reads from this HTTP connection, or,
236         * error stream if the connection failed but the server sent useful data.
237         * <p>
238         * Note: the return value can be null, if both the input and the error stream are null.
239         * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887
240         * @return input or error stream
241         * @throws IOException if any I/O error occurs
242         *
243         * @see HttpURLConnection#getInputStream()
244         * @see HttpURLConnection#getErrorStream()
245         */
246        @SuppressWarnings("resource")
247        public InputStream getContent() throws IOException {
248            InputStream in;
249            try {
250                in = connection.getInputStream();
251            } catch (IOException ioe) {
252                in = connection.getErrorStream();
253            }
254            if (in != null) {
255                in = new ProgressInputStream(in, getContentLength(), monitor);
256                in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in;
257                Compression compression = Compression.NONE;
258                if (uncompress) {
259                    final String contentType = getContentType();
260                    Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType);
261                    compression = Compression.forContentType(contentType);
262                }
263                if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) {
264                    final String contentDisposition = getHeaderField("Content-Disposition");
265                    final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(contentDisposition);
266                    if (matcher.find()) {
267                        Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition);
268                        compression = Compression.byExtension(matcher.group(1));
269                    }
270                }
271                in = compression.getUncompressedInputStream(in);
272            }
273            return in;
274        }
275
276        /**
277         * Returns {@link #getContent()} wrapped in a buffered reader.
278         *
279         * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}.
280         * @return buffered reader
281         * @throws IOException if any I/O error occurs
282         */
283        public BufferedReader getContentReader() throws IOException {
284            return new BufferedReader(
285                    UTFInputStreamReader.create(getContent())
286            );
287        }
288
289        /**
290         * Fetches the HTTP response as String.
291         * @return the response
292         * @throws IOException if any I/O error occurs
293         */
294        @SuppressWarnings("resource")
295        public String fetchContent() throws IOException {
296            try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) {
297                return scanner.hasNext() ? scanner.next() : "";
298            }
299        }
300
301        /**
302         * Gets the response code from this HTTP connection.
303         * @return HTTP response code
304         *
305         * @see HttpURLConnection#getResponseCode()
306         */
307        public int getResponseCode() {
308            return responseCode;
309        }
310
311        /**
312         * Gets the response message from this HTTP connection.
313         * @return HTTP response message
314         *
315         * @see HttpURLConnection#getResponseMessage()
316         * @since 9172
317         */
318        public String getResponseMessage() {
319            return responseMessage;
320        }
321
322        /**
323         * Returns the {@code Content-Encoding} header.
324         * @return {@code Content-Encoding} HTTP header
325         * @see HttpURLConnection#getContentEncoding()
326         */
327        public String getContentEncoding() {
328            return connection.getContentEncoding();
329        }
330
331        /**
332         * Returns the {@code Content-Type} header.
333         * @return {@code Content-Type} HTTP header
334         */
335        public String getContentType() {
336            return connection.getHeaderField("Content-Type");
337        }
338
339        /**
340         * Returns the {@code Expire} header.
341         * @return {@code Expire} HTTP header
342         * @see HttpURLConnection#getExpiration()
343         * @since 9232
344         */
345        public long getExpiration() {
346            return connection.getExpiration();
347        }
348
349        /**
350         * Returns the {@code Last-Modified} header.
351         * @return {@code Last-Modified} HTTP header
352         * @see HttpURLConnection#getLastModified()
353         * @since 9232
354         */
355        public long getLastModified() {
356            return connection.getLastModified();
357        }
358
359        /**
360         * Returns the {@code Content-Length} header.
361         * @return {@code Content-Length} HTTP header
362         * @see HttpURLConnection#getContentLengthLong()
363         */
364        public long getContentLength() {
365            return connection.getContentLengthLong();
366        }
367
368        /**
369         * Returns the value of the named header field.
370         * @param name the name of a header field
371         * @return the value of the named header field, or {@code null} if there is no such field in the header
372         * @see HttpURLConnection#getHeaderField(String)
373         * @since 9172
374         */
375        public String getHeaderField(String name) {
376            return connection.getHeaderField(name);
377        }
378
379        /**
380         * Returns an unmodifiable Map mapping header keys to a List of header values.
381         * @return unmodifiable Map mapping header keys to a List of header values
382         * @see HttpURLConnection#getHeaderFields()
383         * @since 9232
384         */
385        public Map<String, List<String>> getHeaderFields() {
386            return connection.getHeaderFields();
387        }
388
389        /**
390         * @see HttpURLConnection#disconnect()
391         */
392        public void disconnect() {
393            HttpClient.disconnect(connection);
394        }
395    }
396
397    /**
398     * Creates a new instance for the given URL and a {@code GET} request
399     *
400     * @param url the URL
401     * @return a new instance
402     */
403    public static HttpClient create(URL url) {
404        return create(url, "GET");
405    }
406
407    /**
408     * Creates a new instance for the given URL and a {@code GET} request
409     *
410     * @param url the URL
411     * @param requestMethod the HTTP request method to perform when calling
412     * @return a new instance
413     */
414    public static HttpClient create(URL url, String requestMethod) {
415        return new HttpClient(url, requestMethod);
416    }
417
418    /**
419     * Returns the URL set for this connection.
420     * @return the URL
421     * @see #create(URL)
422     * @see #create(URL, String)
423     * @since 9172
424     */
425    public URL getURL() {
426        return url;
427    }
428
429    /**
430     * Returns the request method set for this connection.
431     * @return the HTTP request method
432     * @see #create(URL, String)
433     * @since 9172
434     */
435    public String getRequestMethod() {
436        return requestMethod;
437    }
438
439    /**
440     * Returns the set value for the given {@code header}.
441     * @param header HTTP header name
442     * @return HTTP header value
443     * @since 9172
444     */
445    public String getRequestHeader(String header) {
446        return headers.get(header);
447    }
448
449    /**
450     * Sets whether not to set header {@code Cache-Control=no-cache}
451     *
452     * @param useCache whether not to set header {@code Cache-Control=no-cache}
453     * @return {@code this}
454     * @see HttpURLConnection#setUseCaches(boolean)
455     */
456    public HttpClient useCache(boolean useCache) {
457        this.useCache = useCache;
458        return this;
459    }
460
461    /**
462     * Sets whether not to set header {@code Connection=close}
463     * <p>
464     * This might fix #7640, see
465     * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>.
466     *
467     * @param keepAlive whether not to set header {@code Connection=close}
468     * @return {@code this}
469     */
470    public HttpClient keepAlive(boolean keepAlive) {
471        return setHeader("Connection", keepAlive ? null : "close");
472    }
473
474    /**
475     * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced
476     * by this URLConnection. If the timeout expires before the connection can be established, a
477     * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
478     * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds
479     * @return {@code this}
480     * @see HttpURLConnection#setConnectTimeout(int)
481     */
482    public HttpClient setConnectTimeout(int connectTimeout) {
483        this.connectTimeout = connectTimeout;
484        return this;
485    }
486
487    /**
488     * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from
489     * input stream when a connection is established to a resource. If the timeout expires before there is data available for
490     * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout.
491     * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds
492     * @return {@code this}
493     * @see HttpURLConnection#setReadTimeout(int)
494     */
495    public HttpClient setReadTimeout(int readTimeout) {
496        this.readTimeout = readTimeout;
497        return this;
498    }
499
500    /**
501     * This method is used to enable streaming of a HTTP request body without internal buffering,
502     * when the content length is known in advance.
503     * <p>
504     * An exception will be thrown if the application attempts to write more data than the indicated content-length,
505     * or if the application closes the OutputStream before writing the indicated amount.
506     * <p>
507     * When output streaming is enabled, authentication and redirection cannot be handled automatically.
508     * A {@linkplain HttpRetryException} will be thrown when reading the response if authentication or redirection
509     * are required. This exception can be queried for the details of the error.
510     *
511     * @param contentLength The number of bytes which will be written to the OutputStream
512     * @return {@code this}
513     * @see HttpURLConnection#setFixedLengthStreamingMode(long)
514     * @since 9178
515     * @deprecated Submitting data via POST, PUT, DELETE automatically sets this property on the connection
516     */
517    @Deprecated
518    public HttpClient setFixedLengthStreamingMode(long contentLength) {
519        return this;
520    }
521
522    /**
523     * Sets the {@code Accept} header.
524     * @param accept header value
525     *
526     * @return {@code this}
527     */
528    public HttpClient setAccept(String accept) {
529        return setHeader("Accept", accept);
530    }
531
532    /**
533     * Sets the request body for {@code PUT}/{@code POST} requests.
534     * @param requestBody request body
535     *
536     * @return {@code this}
537     */
538    public HttpClient setRequestBody(byte[] requestBody) {
539        this.requestBody = requestBody;
540        return this;
541    }
542
543    /**
544     * Sets the {@code If-Modified-Since} header.
545     * @param ifModifiedSince header value
546     *
547     * @return {@code this}
548     */
549    public HttpClient setIfModifiedSince(long ifModifiedSince) {
550        this.ifModifiedSince = ifModifiedSince;
551        return this;
552    }
553
554    /**
555     * Sets the maximum number of redirections to follow.
556     *
557     * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e.,
558     * to not throw an {@link IOException} in {@link #connect()}.
559     * @param maxRedirects header value
560     *
561     * @return {@code this}
562     */
563    public HttpClient setMaxRedirects(int maxRedirects) {
564        this.maxRedirects = maxRedirects;
565        return this;
566    }
567
568    /**
569     * Sets an arbitrary HTTP header.
570     * @param key header name
571     * @param value header value
572     *
573     * @return {@code this}
574     */
575    public HttpClient setHeader(String key, String value) {
576        this.headers.put(key, value);
577        return this;
578    }
579
580    /**
581     * Sets arbitrary HTTP headers.
582     * @param headers HTTP headers
583     *
584     * @return {@code this}
585     */
586    public HttpClient setHeaders(Map<String, String> headers) {
587        this.headers.putAll(headers);
588        return this;
589    }
590
591    /**
592     * Sets a reason to show on console. Can be {@code null} if no reason is given.
593     * @param reasonForRequest Reason to show
594     * @return {@code this}
595     * @since 9172
596     */
597    public HttpClient setReasonForRequest(String reasonForRequest) {
598        this.reasonForRequest = reasonForRequest;
599        return this;
600    }
601
602    private static boolean isRedirect(final int statusCode) {
603        switch (statusCode) {
604            case HttpURLConnection.HTTP_MOVED_PERM: // 301
605            case HttpURLConnection.HTTP_MOVED_TEMP: // 302
606            case HttpURLConnection.HTTP_SEE_OTHER: // 303
607            case 307: // TEMPORARY_REDIRECT:
608            case 308: // PERMANENT_REDIRECT:
609                return true;
610            default:
611                return false;
612        }
613    }
614
615    /**
616     * @see HttpURLConnection#disconnect()
617     * @since 9309
618     */
619    public void disconnect() {
620        HttpClient.disconnect(connection);
621    }
622
623    private static void disconnect(final HttpURLConnection connection) {
624        // Fix upload aborts - see #263
625        connection.setConnectTimeout(100);
626        connection.setReadTimeout(100);
627        try {
628            Thread.sleep(100);
629        } catch (InterruptedException ex) {
630            Main.warn("InterruptedException in " + HttpClient.class + " during cancel");
631        }
632        connection.disconnect();
633    }
634}