001 /**
002 res * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.hdfs.web;
020
021 import java.io.BufferedOutputStream;
022 import java.io.FileNotFoundException;
023 import java.io.IOException;
024 import java.io.InputStream;
025 import java.io.InputStreamReader;
026 import java.net.HttpURLConnection;
027 import java.net.InetSocketAddress;
028 import java.net.MalformedURLException;
029 import java.net.URI;
030 import java.net.URL;
031 import java.security.PrivilegedExceptionAction;
032 import java.util.ArrayList;
033 import java.util.EnumSet;
034 import java.util.List;
035 import java.util.Map;
036 import java.util.StringTokenizer;
037
038 import javax.ws.rs.core.MediaType;
039
040 import org.apache.commons.logging.Log;
041 import org.apache.commons.logging.LogFactory;
042 import org.apache.hadoop.conf.Configuration;
043 import org.apache.hadoop.fs.BlockLocation;
044 import org.apache.hadoop.fs.CommonConfigurationKeys;
045 import org.apache.hadoop.fs.ContentSummary;
046 import org.apache.hadoop.fs.DelegationTokenRenewer;
047 import org.apache.hadoop.fs.FSDataInputStream;
048 import org.apache.hadoop.fs.FSDataOutputStream;
049 import org.apache.hadoop.fs.FileStatus;
050 import org.apache.hadoop.fs.FileSystem;
051 import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
052 import org.apache.hadoop.fs.Options;
053 import org.apache.hadoop.fs.Path;
054 import org.apache.hadoop.fs.XAttrCodec;
055 import org.apache.hadoop.fs.XAttrSetFlag;
056 import org.apache.hadoop.fs.permission.AclEntry;
057 import org.apache.hadoop.fs.permission.AclStatus;
058 import org.apache.hadoop.fs.permission.FsAction;
059 import org.apache.hadoop.fs.permission.FsPermission;
060 import org.apache.hadoop.hdfs.DFSConfigKeys;
061 import org.apache.hadoop.hdfs.DFSUtil;
062 import org.apache.hadoop.hdfs.HAUtil;
063 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
064 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
065 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
066 import org.apache.hadoop.hdfs.web.resources.*;
067 import org.apache.hadoop.hdfs.web.resources.HttpOpParam.Op;
068 import org.apache.hadoop.io.Text;
069 import org.apache.hadoop.io.retry.RetryPolicies;
070 import org.apache.hadoop.io.retry.RetryPolicy;
071 import org.apache.hadoop.io.retry.RetryUtils;
072 import org.apache.hadoop.ipc.RemoteException;
073 import org.apache.hadoop.net.NetUtils;
074 import org.apache.hadoop.security.AccessControlException;
075 import org.apache.hadoop.security.SecurityUtil;
076 import org.apache.hadoop.security.UserGroupInformation;
077 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
078 import org.apache.hadoop.security.token.Token;
079 import org.apache.hadoop.security.token.TokenIdentifier;
080 import org.apache.hadoop.security.token.TokenSelector;
081 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
082 import org.apache.hadoop.util.Progressable;
083 import org.mortbay.util.ajax.JSON;
084
085 import com.google.common.annotations.VisibleForTesting;
086 import com.google.common.base.Charsets;
087 import com.google.common.base.Preconditions;
088 import com.google.common.collect.Lists;
089
090 /** A FileSystem for HDFS over the web. */
091 public class WebHdfsFileSystem extends FileSystem
092 implements DelegationTokenRenewer.Renewable, TokenAspect.TokenManagementDelegator {
093 public static final Log LOG = LogFactory.getLog(WebHdfsFileSystem.class);
094 /** File System URI: {SCHEME}://namenode:port/path/to/file */
095 public static final String SCHEME = "webhdfs";
096 /** WebHdfs version. */
097 public static final int VERSION = 1;
098 /** Http URI: http://namenode:port/{PATH_PREFIX}/path/to/file */
099 public static final String PATH_PREFIX = "/" + SCHEME + "/v" + VERSION;
100
101 /** Default connection factory may be overridden in tests to use smaller timeout values */
102 protected URLConnectionFactory connectionFactory;
103
104 /** Delegation token kind */
105 public static final Text TOKEN_KIND = new Text("WEBHDFS delegation");
106
107 @VisibleForTesting
108 public static final String CANT_FALLBACK_TO_INSECURE_MSG =
109 "The client is configured to only allow connecting to secure cluster";
110
111 private boolean canRefreshDelegationToken;
112
113 private UserGroupInformation ugi;
114 private URI uri;
115 private Token<?> delegationToken;
116 protected Text tokenServiceName;
117 private RetryPolicy retryPolicy = null;
118 private Path workingDir;
119 private InetSocketAddress nnAddrs[];
120 private int currentNNAddrIndex;
121 private boolean disallowFallbackToInsecureCluster;
122
123 /**
124 * Return the protocol scheme for the FileSystem.
125 * <p/>
126 *
127 * @return <code>webhdfs</code>
128 */
129 @Override
130 public String getScheme() {
131 return SCHEME;
132 }
133
134 /**
135 * return the underlying transport protocol (http / https).
136 */
137 protected String getTransportScheme() {
138 return "http";
139 }
140
141 protected Text getTokenKind() {
142 return TOKEN_KIND;
143 }
144
145 @Override
146 public synchronized void initialize(URI uri, Configuration conf
147 ) throws IOException {
148 super.initialize(uri, conf);
149 setConf(conf);
150 /** set user pattern based on configuration file */
151 UserParam.setUserPattern(conf.get(
152 DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
153 DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
154
155 connectionFactory = URLConnectionFactory
156 .newDefaultURLConnectionFactory(conf);
157
158 ugi = UserGroupInformation.getCurrentUser();
159 this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
160 this.nnAddrs = resolveNNAddr();
161
162 boolean isHA = HAUtil.isClientFailoverConfigured(conf, this.uri);
163 boolean isLogicalUri = isHA && HAUtil.isLogicalUri(conf, this.uri);
164 // In non-HA or non-logical URI case, the code needs to call
165 // getCanonicalUri() in order to handle the case where no port is
166 // specified in the URI
167 this.tokenServiceName = isLogicalUri ?
168 HAUtil.buildTokenServiceForLogicalUri(uri, getScheme())
169 : SecurityUtil.buildTokenService(getCanonicalUri());
170
171 if (!isHA) {
172 this.retryPolicy =
173 RetryUtils.getDefaultRetryPolicy(
174 conf,
175 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_KEY,
176 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
177 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_KEY,
178 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
179 SafeModeException.class);
180 } else {
181
182 int maxFailoverAttempts = conf.getInt(
183 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
184 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
185 int maxRetryAttempts = conf.getInt(
186 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
187 DFSConfigKeys.DFS_HTTP_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
188 int failoverSleepBaseMillis = conf.getInt(
189 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
190 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
191 int failoverSleepMaxMillis = conf.getInt(
192 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
193 DFSConfigKeys.DFS_HTTP_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
194
195 this.retryPolicy = RetryPolicies
196 .failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,
197 maxFailoverAttempts, maxRetryAttempts, failoverSleepBaseMillis,
198 failoverSleepMaxMillis);
199 }
200
201 this.workingDir = getHomeDirectory();
202 this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled();
203 this.disallowFallbackToInsecureCluster = !conf.getBoolean(
204 CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
205 CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT);
206 this.delegationToken = null;
207 }
208
209 @Override
210 public URI getCanonicalUri() {
211 return super.getCanonicalUri();
212 }
213
214 /** Is WebHDFS enabled in conf? */
215 public static boolean isEnabled(final Configuration conf, final Log log) {
216 final boolean b = conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
217 DFSConfigKeys.DFS_WEBHDFS_ENABLED_DEFAULT);
218 return b;
219 }
220
221 TokenSelector<DelegationTokenIdentifier> tokenSelector =
222 new AbstractDelegationTokenSelector<DelegationTokenIdentifier>(getTokenKind()){};
223
224 // the first getAuthParams() for a non-token op will either get the
225 // internal token from the ugi or lazy fetch one
226 protected synchronized Token<?> getDelegationToken() throws IOException {
227 if (canRefreshDelegationToken && delegationToken == null) {
228 Token<?> token = tokenSelector.selectToken(
229 new Text(getCanonicalServiceName()), ugi.getTokens());
230 // ugi tokens are usually indicative of a task which can't
231 // refetch tokens. even if ugi has credentials, don't attempt
232 // to get another token to match hdfs/rpc behavior
233 if (token != null) {
234 LOG.debug("Using UGI token: " + token);
235 canRefreshDelegationToken = false;
236 } else {
237 token = getDelegationToken(null);
238 if (token != null) {
239 LOG.debug("Fetched new token: " + token);
240 } else { // security is disabled
241 canRefreshDelegationToken = false;
242 }
243 }
244 setDelegationToken(token);
245 }
246 return delegationToken;
247 }
248
249 @VisibleForTesting
250 synchronized boolean replaceExpiredDelegationToken() throws IOException {
251 boolean replaced = false;
252 if (canRefreshDelegationToken) {
253 Token<?> token = getDelegationToken(null);
254 LOG.debug("Replaced expired token: " + token);
255 setDelegationToken(token);
256 replaced = (token != null);
257 }
258 return replaced;
259 }
260
261 @Override
262 @VisibleForTesting
263 public int getDefaultPort() {
264 return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
265 DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
266 }
267
268 @Override
269 public URI getUri() {
270 return this.uri;
271 }
272
273 @Override
274 protected URI canonicalizeUri(URI uri) {
275 return NetUtils.getCanonicalUri(uri, getDefaultPort());
276 }
277
278 /** @return the home directory. */
279 public static String getHomeDirectoryString(final UserGroupInformation ugi) {
280 return "/user/" + ugi.getShortUserName();
281 }
282
283 @Override
284 public Path getHomeDirectory() {
285 return makeQualified(new Path(getHomeDirectoryString(ugi)));
286 }
287
288 @Override
289 public synchronized Path getWorkingDirectory() {
290 return workingDir;
291 }
292
293 @Override
294 public synchronized void setWorkingDirectory(final Path dir) {
295 String result = makeAbsolute(dir).toUri().getPath();
296 if (!DFSUtil.isValidName(result)) {
297 throw new IllegalArgumentException("Invalid DFS directory name " +
298 result);
299 }
300 workingDir = makeAbsolute(dir);
301 }
302
303 private Path makeAbsolute(Path f) {
304 return f.isAbsolute()? f: new Path(workingDir, f);
305 }
306
307 static Map<?, ?> jsonParse(final HttpURLConnection c, final boolean useErrorStream
308 ) throws IOException {
309 if (c.getContentLength() == 0) {
310 return null;
311 }
312 final InputStream in = useErrorStream? c.getErrorStream(): c.getInputStream();
313 if (in == null) {
314 throw new IOException("The " + (useErrorStream? "error": "input") + " stream is null.");
315 }
316 final String contentType = c.getContentType();
317 if (contentType != null) {
318 final MediaType parsed = MediaType.valueOf(contentType);
319 if (!MediaType.APPLICATION_JSON_TYPE.isCompatible(parsed)) {
320 throw new IOException("Content-Type \"" + contentType
321 + "\" is incompatible with \"" + MediaType.APPLICATION_JSON
322 + "\" (parsed=\"" + parsed + "\")");
323 }
324 }
325 return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
326 }
327
328 private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
329 final HttpURLConnection conn, boolean unwrapException) throws IOException {
330 final int code = conn.getResponseCode();
331 // server is demanding an authentication we don't support
332 if (code == HttpURLConnection.HTTP_UNAUTHORIZED) {
333 // match hdfs/rpc exception
334 throw new AccessControlException(conn.getResponseMessage());
335 }
336 if (code != op.getExpectedHttpResponseCode()) {
337 final Map<?, ?> m;
338 try {
339 m = jsonParse(conn, true);
340 } catch(Exception e) {
341 throw new IOException("Unexpected HTTP response: code=" + code + " != "
342 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
343 + ", message=" + conn.getResponseMessage(), e);
344 }
345
346 if (m == null) {
347 throw new IOException("Unexpected HTTP response: code=" + code + " != "
348 + op.getExpectedHttpResponseCode() + ", " + op.toQueryString()
349 + ", message=" + conn.getResponseMessage());
350 } else if (m.get(RemoteException.class.getSimpleName()) == null) {
351 return m;
352 }
353
354 IOException re = JsonUtil.toRemoteException(m);
355 // extract UGI-related exceptions and unwrap InvalidToken
356 // the NN mangles these exceptions but the DN does not and may need
357 // to re-fetch a token if either report the token is expired
358 if (re.getMessage() != null && re.getMessage().startsWith("Failed to obtain user group information:")) {
359 String[] parts = re.getMessage().split(":\\s+", 3);
360 re = new RemoteException(parts[1], parts[2]);
361 re = ((RemoteException)re).unwrapRemoteException(InvalidToken.class);
362 }
363 throw unwrapException? toIOException(re): re;
364 }
365 return null;
366 }
367
368 /**
369 * Covert an exception to an IOException.
370 *
371 * For a non-IOException, wrap it with IOException.
372 * For a RemoteException, unwrap it.
373 * For an IOException which is not a RemoteException, return it.
374 */
375 private static IOException toIOException(Exception e) {
376 if (!(e instanceof IOException)) {
377 return new IOException(e);
378 }
379
380 final IOException ioe = (IOException)e;
381 if (!(ioe instanceof RemoteException)) {
382 return ioe;
383 }
384
385 return ((RemoteException)ioe).unwrapRemoteException();
386 }
387
388 private synchronized InetSocketAddress getCurrentNNAddr() {
389 return nnAddrs[currentNNAddrIndex];
390 }
391
392 /**
393 * Reset the appropriate state to gracefully fail over to another name node
394 */
395 private synchronized void resetStateToFailOver() {
396 currentNNAddrIndex = (currentNNAddrIndex + 1) % nnAddrs.length;
397 }
398
399 /**
400 * Return a URL pointing to given path on the namenode.
401 *
402 * @param path to obtain the URL for
403 * @param query string to append to the path
404 * @return namenode URL referring to the given path
405 * @throws IOException on error constructing the URL
406 */
407 private URL getNamenodeURL(String path, String query) throws IOException {
408 InetSocketAddress nnAddr = getCurrentNNAddr();
409 final URL url = new URL(getTransportScheme(), nnAddr.getHostName(),
410 nnAddr.getPort(), path + '?' + query);
411 if (LOG.isTraceEnabled()) {
412 LOG.trace("url=" + url);
413 }
414 return url;
415 }
416
417 Param<?,?>[] getAuthParameters(final HttpOpParam.Op op) throws IOException {
418 List<Param<?,?>> authParams = Lists.newArrayList();
419 // Skip adding delegation token for token operations because these
420 // operations require authentication.
421 Token<?> token = null;
422 if (!op.getRequireAuth()) {
423 token = getDelegationToken();
424 }
425 if (token != null) {
426 authParams.add(new DelegationParam(token.encodeToUrlString()));
427 } else {
428 UserGroupInformation userUgi = ugi;
429 UserGroupInformation realUgi = userUgi.getRealUser();
430 if (realUgi != null) { // proxy user
431 authParams.add(new DoAsParam(userUgi.getShortUserName()));
432 userUgi = realUgi;
433 }
434 authParams.add(new UserParam(userUgi.getShortUserName()));
435 }
436 return authParams.toArray(new Param<?,?>[0]);
437 }
438
439 URL toUrl(final HttpOpParam.Op op, final Path fspath,
440 final Param<?,?>... parameters) throws IOException {
441 //initialize URI path and query
442 final String path = PATH_PREFIX
443 + (fspath == null? "/": makeQualified(fspath).toUri().getRawPath());
444 final String query = op.toQueryString()
445 + Param.toSortedString("&", getAuthParameters(op))
446 + Param.toSortedString("&", parameters);
447 final URL url = getNamenodeURL(path, query);
448 if (LOG.isTraceEnabled()) {
449 LOG.trace("url=" + url);
450 }
451 return url;
452 }
453
454 /**
455 * This class is for initialing a HTTP connection, connecting to server,
456 * obtaining a response, and also handling retry on failures.
457 */
458 abstract class AbstractRunner<T> {
459 abstract protected URL getUrl() throws IOException;
460
461 protected final HttpOpParam.Op op;
462 private final boolean redirected;
463 protected ExcludeDatanodesParam excludeDatanodes = new ExcludeDatanodesParam("");
464
465 private boolean checkRetry;
466
467 protected AbstractRunner(final HttpOpParam.Op op, boolean redirected) {
468 this.op = op;
469 this.redirected = redirected;
470 }
471
472 T run() throws IOException {
473 UserGroupInformation connectUgi = ugi.getRealUser();
474 if (connectUgi == null) {
475 connectUgi = ugi;
476 }
477 if (op.getRequireAuth()) {
478 connectUgi.checkTGTAndReloginFromKeytab();
479 }
480 try {
481 // the entire lifecycle of the connection must be run inside the
482 // doAs to ensure authentication is performed correctly
483 return connectUgi.doAs(
484 new PrivilegedExceptionAction<T>() {
485 @Override
486 public T run() throws IOException {
487 return runWithRetry();
488 }
489 });
490 } catch (InterruptedException e) {
491 throw new IOException(e);
492 }
493 }
494
495 /**
496 * Two-step requests redirected to a DN
497 *
498 * Create/Append:
499 * Step 1) Submit a Http request with neither auto-redirect nor data.
500 * Step 2) Submit another Http request with the URL from the Location header with data.
501 *
502 * The reason of having two-step create/append is for preventing clients to
503 * send out the data before the redirect. This issue is addressed by the
504 * "Expect: 100-continue" header in HTTP/1.1; see RFC 2616, Section 8.2.3.
505 * Unfortunately, there are software library bugs (e.g. Jetty 6 http server
506 * and Java 6 http client), which do not correctly implement "Expect:
507 * 100-continue". The two-step create/append is a temporary workaround for
508 * the software library bugs.
509 *
510 * Open/Checksum
511 * Also implements two-step connects for other operations redirected to
512 * a DN such as open and checksum
513 */
514 private HttpURLConnection connect(URL url) throws IOException {
515 //redirect hostname and port
516 String redirectHost = null;
517
518
519 // resolve redirects for a DN operation unless already resolved
520 if (op.getRedirect() && !redirected) {
521 final HttpOpParam.Op redirectOp =
522 HttpOpParam.TemporaryRedirectOp.valueOf(op);
523 final HttpURLConnection conn = connect(redirectOp, url);
524 // application level proxy like httpfs might not issue a redirect
525 if (conn.getResponseCode() == op.getExpectedHttpResponseCode()) {
526 return conn;
527 }
528 try {
529 validateResponse(redirectOp, conn, false);
530 url = new URL(conn.getHeaderField("Location"));
531 redirectHost = url.getHost() + ":" + url.getPort();
532 } finally {
533 conn.disconnect();
534 }
535 }
536 try {
537 return connect(op, url);
538 } catch (IOException ioe) {
539 if (redirectHost != null) {
540 if (excludeDatanodes.getValue() != null) {
541 excludeDatanodes = new ExcludeDatanodesParam(redirectHost + ","
542 + excludeDatanodes.getValue());
543 } else {
544 excludeDatanodes = new ExcludeDatanodesParam(redirectHost);
545 }
546 }
547 throw ioe;
548 }
549 }
550
551 private HttpURLConnection connect(final HttpOpParam.Op op, final URL url)
552 throws IOException {
553 final HttpURLConnection conn =
554 (HttpURLConnection)connectionFactory.openConnection(url);
555 final boolean doOutput = op.getDoOutput();
556 conn.setRequestMethod(op.getType().toString());
557 conn.setInstanceFollowRedirects(false);
558 switch (op.getType()) {
559 // if not sending a message body for a POST or PUT operation, need
560 // to ensure the server/proxy knows this
561 case POST:
562 case PUT: {
563 conn.setDoOutput(true);
564 if (!doOutput) {
565 // explicitly setting content-length to 0 won't do spnego!!
566 // opening and closing the stream will send "Content-Length: 0"
567 conn.getOutputStream().close();
568 } else {
569 conn.setRequestProperty("Content-Type",
570 MediaType.APPLICATION_OCTET_STREAM);
571 conn.setChunkedStreamingMode(32 << 10); //32kB-chunk
572 }
573 break;
574 }
575 default: {
576 conn.setDoOutput(doOutput);
577 break;
578 }
579 }
580 conn.connect();
581 return conn;
582 }
583
584 private T runWithRetry() throws IOException {
585 /**
586 * Do the real work.
587 *
588 * There are three cases that the code inside the loop can throw an
589 * IOException:
590 *
591 * <ul>
592 * <li>The connection has failed (e.g., ConnectException,
593 * @see FailoverOnNetworkExceptionRetry for more details)</li>
594 * <li>The namenode enters the standby state (i.e., StandbyException).</li>
595 * <li>The server returns errors for the command (i.e., RemoteException)</li>
596 * </ul>
597 *
598 * The call to shouldRetry() will conduct the retry policy. The policy
599 * examines the exception and swallows it if it decides to rerun the work.
600 */
601 for(int retry = 0; ; retry++) {
602 checkRetry = !redirected;
603 final URL url = getUrl();
604 try {
605 final HttpURLConnection conn = connect(url);
606 // output streams will validate on close
607 if (!op.getDoOutput()) {
608 validateResponse(op, conn, false);
609 }
610 return getResponse(conn);
611 } catch (AccessControlException ace) {
612 // no retries for auth failures
613 throw ace;
614 } catch (InvalidToken it) {
615 // try to replace the expired token with a new one. the attempt
616 // to acquire a new token must be outside this operation's retry
617 // so if it fails after its own retries, this operation fails too.
618 if (op.getRequireAuth() || !replaceExpiredDelegationToken()) {
619 throw it;
620 }
621 } catch (IOException ioe) {
622 shouldRetry(ioe, retry);
623 }
624 }
625 }
626
627 private void shouldRetry(final IOException ioe, final int retry
628 ) throws IOException {
629 InetSocketAddress nnAddr = getCurrentNNAddr();
630 if (checkRetry) {
631 try {
632 final RetryPolicy.RetryAction a = retryPolicy.shouldRetry(
633 ioe, retry, 0, true);
634
635 boolean isRetry = a.action == RetryPolicy.RetryAction.RetryDecision.RETRY;
636 boolean isFailoverAndRetry =
637 a.action == RetryPolicy.RetryAction.RetryDecision.FAILOVER_AND_RETRY;
638
639 if (isRetry || isFailoverAndRetry) {
640 LOG.info("Retrying connect to namenode: " + nnAddr
641 + ". Already tried " + retry + " time(s); retry policy is "
642 + retryPolicy + ", delay " + a.delayMillis + "ms.");
643
644 if (isFailoverAndRetry) {
645 resetStateToFailOver();
646 }
647
648 Thread.sleep(a.delayMillis);
649 return;
650 }
651 } catch(Exception e) {
652 LOG.warn("Original exception is ", ioe);
653 throw toIOException(e);
654 }
655 }
656 throw toIOException(ioe);
657 }
658
659 abstract T getResponse(HttpURLConnection conn) throws IOException;
660 }
661
662 /**
663 * Abstract base class to handle path-based operations with params
664 */
665 abstract class AbstractFsPathRunner<T> extends AbstractRunner<T> {
666 private final Path fspath;
667 private final Param<?,?>[] parameters;
668
669 AbstractFsPathRunner(final HttpOpParam.Op op, final Path fspath,
670 Param<?,?>... parameters) {
671 super(op, false);
672 this.fspath = fspath;
673 this.parameters = parameters;
674 }
675
676 AbstractFsPathRunner(final HttpOpParam.Op op, Param<?,?>[] parameters,
677 final Path fspath) {
678 super(op, false);
679 this.fspath = fspath;
680 this.parameters = parameters;
681 }
682
683 @Override
684 protected URL getUrl() throws IOException {
685 if (excludeDatanodes.getValue() != null) {
686 Param<?, ?>[] tmpParam = new Param<?, ?>[parameters.length + 1];
687 System.arraycopy(parameters, 0, tmpParam, 0, parameters.length);
688 tmpParam[parameters.length] = excludeDatanodes;
689 return toUrl(op, fspath, tmpParam);
690 } else {
691 return toUrl(op, fspath, parameters);
692 }
693 }
694 }
695
696 /**
697 * Default path-based implementation expects no json response
698 */
699 class FsPathRunner extends AbstractFsPathRunner<Void> {
700 FsPathRunner(Op op, Path fspath, Param<?,?>... parameters) {
701 super(op, fspath, parameters);
702 }
703
704 @Override
705 Void getResponse(HttpURLConnection conn) throws IOException {
706 return null;
707 }
708 }
709
710 /**
711 * Handle path-based operations with a json response
712 */
713 abstract class FsPathResponseRunner<T> extends AbstractFsPathRunner<T> {
714 FsPathResponseRunner(final HttpOpParam.Op op, final Path fspath,
715 Param<?,?>... parameters) {
716 super(op, fspath, parameters);
717 }
718
719 FsPathResponseRunner(final HttpOpParam.Op op, Param<?,?>[] parameters,
720 final Path fspath) {
721 super(op, parameters, fspath);
722 }
723
724 @Override
725 final T getResponse(HttpURLConnection conn) throws IOException {
726 try {
727 final Map<?,?> json = jsonParse(conn, false);
728 if (json == null) {
729 // match exception class thrown by parser
730 throw new IllegalStateException("Missing response");
731 }
732 return decodeResponse(json);
733 } catch (IOException ioe) {
734 throw ioe;
735 } catch (Exception e) { // catch json parser errors
736 final IOException ioe =
737 new IOException("Response decoding failure: "+e.toString(), e);
738 if (LOG.isDebugEnabled()) {
739 LOG.debug(ioe);
740 }
741 throw ioe;
742 } finally {
743 conn.disconnect();
744 }
745 }
746
747 abstract T decodeResponse(Map<?,?> json) throws IOException;
748 }
749
750 /**
751 * Handle path-based operations with json boolean response
752 */
753 class FsPathBooleanRunner extends FsPathResponseRunner<Boolean> {
754 FsPathBooleanRunner(Op op, Path fspath, Param<?,?>... parameters) {
755 super(op, fspath, parameters);
756 }
757
758 @Override
759 Boolean decodeResponse(Map<?,?> json) throws IOException {
760 return (Boolean)json.get("boolean");
761 }
762 }
763
764 /**
765 * Handle create/append output streams
766 */
767 class FsPathOutputStreamRunner extends AbstractFsPathRunner<FSDataOutputStream> {
768 private final int bufferSize;
769
770 FsPathOutputStreamRunner(Op op, Path fspath, int bufferSize,
771 Param<?,?>... parameters) {
772 super(op, fspath, parameters);
773 this.bufferSize = bufferSize;
774 }
775
776 @Override
777 FSDataOutputStream getResponse(final HttpURLConnection conn)
778 throws IOException {
779 return new FSDataOutputStream(new BufferedOutputStream(
780 conn.getOutputStream(), bufferSize), statistics) {
781 @Override
782 public void close() throws IOException {
783 try {
784 super.close();
785 } finally {
786 try {
787 validateResponse(op, conn, true);
788 } finally {
789 conn.disconnect();
790 }
791 }
792 }
793 };
794 }
795 }
796
797 class FsPathConnectionRunner extends AbstractFsPathRunner<HttpURLConnection> {
798 FsPathConnectionRunner(Op op, Path fspath, Param<?,?>... parameters) {
799 super(op, fspath, parameters);
800 }
801 @Override
802 HttpURLConnection getResponse(final HttpURLConnection conn)
803 throws IOException {
804 return conn;
805 }
806 }
807
808 /**
809 * Used by open() which tracks the resolved url itself
810 */
811 final class URLRunner extends AbstractRunner<HttpURLConnection> {
812 private final URL url;
813 @Override
814 protected URL getUrl() {
815 return url;
816 }
817
818 protected URLRunner(final HttpOpParam.Op op, final URL url, boolean redirected) {
819 super(op, redirected);
820 this.url = url;
821 }
822
823 @Override
824 HttpURLConnection getResponse(HttpURLConnection conn) throws IOException {
825 return conn;
826 }
827 }
828
829 private FsPermission applyUMask(FsPermission permission) {
830 if (permission == null) {
831 permission = FsPermission.getDefault();
832 }
833 return permission.applyUMask(FsPermission.getUMask(getConf()));
834 }
835
836 private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
837 final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
838 HdfsFileStatus status = new FsPathResponseRunner<HdfsFileStatus>(op, f) {
839 @Override
840 HdfsFileStatus decodeResponse(Map<?,?> json) {
841 return JsonUtil.toFileStatus(json, true);
842 }
843 }.run();
844 if (status == null) {
845 throw new FileNotFoundException("File does not exist: " + f);
846 }
847 return status;
848 }
849
850 @Override
851 public FileStatus getFileStatus(Path f) throws IOException {
852 statistics.incrementReadOps(1);
853 return makeQualified(getHdfsFileStatus(f), f);
854 }
855
856 private FileStatus makeQualified(HdfsFileStatus f, Path parent) {
857 return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
858 f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
859 f.getPermission(), f.getOwner(), f.getGroup(),
860 f.isSymlink() ? new Path(f.getSymlink()) : null,
861 f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory()));
862 }
863
864 @Override
865 public AclStatus getAclStatus(Path f) throws IOException {
866 final HttpOpParam.Op op = GetOpParam.Op.GETACLSTATUS;
867 AclStatus status = new FsPathResponseRunner<AclStatus>(op, f) {
868 @Override
869 AclStatus decodeResponse(Map<?,?> json) {
870 return JsonUtil.toAclStatus(json);
871 }
872 }.run();
873 if (status == null) {
874 throw new FileNotFoundException("File does not exist: " + f);
875 }
876 return status;
877 }
878
879 @Override
880 public boolean mkdirs(Path f, FsPermission permission) throws IOException {
881 statistics.incrementWriteOps(1);
882 final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
883 return new FsPathBooleanRunner(op, f,
884 new PermissionParam(applyUMask(permission))
885 ).run();
886 }
887
888 /**
889 * Create a symlink pointing to the destination path.
890 * @see org.apache.hadoop.fs.Hdfs#createSymlink(Path, Path, boolean)
891 */
892 public void createSymlink(Path destination, Path f, boolean createParent
893 ) throws IOException {
894 statistics.incrementWriteOps(1);
895 final HttpOpParam.Op op = PutOpParam.Op.CREATESYMLINK;
896 new FsPathRunner(op, f,
897 new DestinationParam(makeQualified(destination).toUri().getPath()),
898 new CreateParentParam(createParent)
899 ).run();
900 }
901
902 @Override
903 public boolean rename(final Path src, final Path dst) throws IOException {
904 statistics.incrementWriteOps(1);
905 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
906 return new FsPathBooleanRunner(op, src,
907 new DestinationParam(makeQualified(dst).toUri().getPath())
908 ).run();
909 }
910
911 @SuppressWarnings("deprecation")
912 @Override
913 public void rename(final Path src, final Path dst,
914 final Options.Rename... options) throws IOException {
915 statistics.incrementWriteOps(1);
916 final HttpOpParam.Op op = PutOpParam.Op.RENAME;
917 new FsPathRunner(op, src,
918 new DestinationParam(makeQualified(dst).toUri().getPath()),
919 new RenameOptionSetParam(options)
920 ).run();
921 }
922
923 @Override
924 public void setXAttr(Path p, String name, byte[] value,
925 EnumSet<XAttrSetFlag> flag) throws IOException {
926 statistics.incrementWriteOps(1);
927 final HttpOpParam.Op op = PutOpParam.Op.SETXATTR;
928 if (value != null) {
929 new FsPathRunner(op, p, new XAttrNameParam(name), new XAttrValueParam(
930 XAttrCodec.encodeValue(value, XAttrCodec.HEX)),
931 new XAttrSetFlagParam(flag)).run();
932 } else {
933 new FsPathRunner(op, p, new XAttrNameParam(name),
934 new XAttrSetFlagParam(flag)).run();
935 }
936 }
937
938 @Override
939 public byte[] getXAttr(Path p, final String name) throws IOException {
940 final HttpOpParam.Op op = GetOpParam.Op.GETXATTRS;
941 return new FsPathResponseRunner<byte[]>(op, p, new XAttrNameParam(name),
942 new XAttrEncodingParam(XAttrCodec.HEX)) {
943 @Override
944 byte[] decodeResponse(Map<?, ?> json) throws IOException {
945 return JsonUtil.getXAttr(json, name);
946 }
947 }.run();
948 }
949
950 @Override
951 public Map<String, byte[]> getXAttrs(Path p) throws IOException {
952 final HttpOpParam.Op op = GetOpParam.Op.GETXATTRS;
953 return new FsPathResponseRunner<Map<String, byte[]>>(op, p,
954 new XAttrEncodingParam(XAttrCodec.HEX)) {
955 @Override
956 Map<String, byte[]> decodeResponse(Map<?, ?> json) throws IOException {
957 return JsonUtil.toXAttrs(json);
958 }
959 }.run();
960 }
961
962 @Override
963 public Map<String, byte[]> getXAttrs(Path p, final List<String> names)
964 throws IOException {
965 Preconditions.checkArgument(names != null && !names.isEmpty(),
966 "XAttr names cannot be null or empty.");
967 Param<?,?>[] parameters = new Param<?,?>[names.size() + 1];
968 for (int i = 0; i < parameters.length - 1; i++) {
969 parameters[i] = new XAttrNameParam(names.get(i));
970 }
971 parameters[parameters.length - 1] = new XAttrEncodingParam(XAttrCodec.HEX);
972
973 final HttpOpParam.Op op = GetOpParam.Op.GETXATTRS;
974 return new FsPathResponseRunner<Map<String, byte[]>>(op, parameters, p) {
975 @Override
976 Map<String, byte[]> decodeResponse(Map<?, ?> json) throws IOException {
977 return JsonUtil.toXAttrs(json);
978 }
979 }.run();
980 }
981
982 @Override
983 public List<String> listXAttrs(Path p) throws IOException {
984 final HttpOpParam.Op op = GetOpParam.Op.LISTXATTRS;
985 return new FsPathResponseRunner<List<String>>(op, p) {
986 @Override
987 List<String> decodeResponse(Map<?, ?> json) throws IOException {
988 return JsonUtil.toXAttrNames(json);
989 }
990 }.run();
991 }
992
993 @Override
994 public void removeXAttr(Path p, String name) throws IOException {
995 statistics.incrementWriteOps(1);
996 final HttpOpParam.Op op = PutOpParam.Op.REMOVEXATTR;
997 new FsPathRunner(op, p, new XAttrNameParam(name)).run();
998 }
999
1000 @Override
1001 public void setOwner(final Path p, final String owner, final String group
1002 ) throws IOException {
1003 if (owner == null && group == null) {
1004 throw new IOException("owner == null && group == null");
1005 }
1006
1007 statistics.incrementWriteOps(1);
1008 final HttpOpParam.Op op = PutOpParam.Op.SETOWNER;
1009 new FsPathRunner(op, p,
1010 new OwnerParam(owner), new GroupParam(group)
1011 ).run();
1012 }
1013
1014 @Override
1015 public void setPermission(final Path p, final FsPermission permission
1016 ) throws IOException {
1017 statistics.incrementWriteOps(1);
1018 final HttpOpParam.Op op = PutOpParam.Op.SETPERMISSION;
1019 new FsPathRunner(op, p,new PermissionParam(permission)).run();
1020 }
1021
1022 @Override
1023 public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
1024 throws IOException {
1025 statistics.incrementWriteOps(1);
1026 final HttpOpParam.Op op = PutOpParam.Op.MODIFYACLENTRIES;
1027 new FsPathRunner(op, path, new AclPermissionParam(aclSpec)).run();
1028 }
1029
1030 @Override
1031 public void removeAclEntries(Path path, List<AclEntry> aclSpec)
1032 throws IOException {
1033 statistics.incrementWriteOps(1);
1034 final HttpOpParam.Op op = PutOpParam.Op.REMOVEACLENTRIES;
1035 new FsPathRunner(op, path, new AclPermissionParam(aclSpec)).run();
1036 }
1037
1038 @Override
1039 public void removeDefaultAcl(Path path) throws IOException {
1040 statistics.incrementWriteOps(1);
1041 final HttpOpParam.Op op = PutOpParam.Op.REMOVEDEFAULTACL;
1042 new FsPathRunner(op, path).run();
1043 }
1044
1045 @Override
1046 public void removeAcl(Path path) throws IOException {
1047 statistics.incrementWriteOps(1);
1048 final HttpOpParam.Op op = PutOpParam.Op.REMOVEACL;
1049 new FsPathRunner(op, path).run();
1050 }
1051
1052 @Override
1053 public void setAcl(final Path p, final List<AclEntry> aclSpec)
1054 throws IOException {
1055 statistics.incrementWriteOps(1);
1056 final HttpOpParam.Op op = PutOpParam.Op.SETACL;
1057 new FsPathRunner(op, p, new AclPermissionParam(aclSpec)).run();
1058 }
1059
1060 @Override
1061 public Path createSnapshot(final Path path, final String snapshotName)
1062 throws IOException {
1063 statistics.incrementWriteOps(1);
1064 final HttpOpParam.Op op = PutOpParam.Op.CREATESNAPSHOT;
1065 Path spath = new FsPathResponseRunner<Path>(op, path,
1066 new SnapshotNameParam(snapshotName)) {
1067 @Override
1068 Path decodeResponse(Map<?,?> json) {
1069 return new Path((String) json.get(Path.class.getSimpleName()));
1070 }
1071 }.run();
1072 return spath;
1073 }
1074
1075 @Override
1076 public void deleteSnapshot(final Path path, final String snapshotName)
1077 throws IOException {
1078 statistics.incrementWriteOps(1);
1079 final HttpOpParam.Op op = DeleteOpParam.Op.DELETESNAPSHOT;
1080 new FsPathRunner(op, path, new SnapshotNameParam(snapshotName)).run();
1081 }
1082
1083 @Override
1084 public void renameSnapshot(final Path path, final String snapshotOldName,
1085 final String snapshotNewName) throws IOException {
1086 statistics.incrementWriteOps(1);
1087 final HttpOpParam.Op op = PutOpParam.Op.RENAMESNAPSHOT;
1088 new FsPathRunner(op, path, new OldSnapshotNameParam(snapshotOldName),
1089 new SnapshotNameParam(snapshotNewName)).run();
1090 }
1091
1092 @Override
1093 public boolean setReplication(final Path p, final short replication
1094 ) throws IOException {
1095 statistics.incrementWriteOps(1);
1096 final HttpOpParam.Op op = PutOpParam.Op.SETREPLICATION;
1097 return new FsPathBooleanRunner(op, p,
1098 new ReplicationParam(replication)
1099 ).run();
1100 }
1101
1102 @Override
1103 public void setTimes(final Path p, final long mtime, final long atime
1104 ) throws IOException {
1105 statistics.incrementWriteOps(1);
1106 final HttpOpParam.Op op = PutOpParam.Op.SETTIMES;
1107 new FsPathRunner(op, p,
1108 new ModificationTimeParam(mtime),
1109 new AccessTimeParam(atime)
1110 ).run();
1111 }
1112
1113 @Override
1114 public long getDefaultBlockSize() {
1115 return getConf().getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
1116 DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
1117 }
1118
1119 @Override
1120 public short getDefaultReplication() {
1121 return (short)getConf().getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
1122 DFSConfigKeys.DFS_REPLICATION_DEFAULT);
1123 }
1124
1125 @Override
1126 public void concat(final Path trg, final Path [] srcs) throws IOException {
1127 statistics.incrementWriteOps(1);
1128 final HttpOpParam.Op op = PostOpParam.Op.CONCAT;
1129 new FsPathRunner(op, trg, new ConcatSourcesParam(srcs)).run();
1130 }
1131
1132 @Override
1133 public FSDataOutputStream create(final Path f, final FsPermission permission,
1134 final boolean overwrite, final int bufferSize, final short replication,
1135 final long blockSize, final Progressable progress) throws IOException {
1136 statistics.incrementWriteOps(1);
1137
1138 final HttpOpParam.Op op = PutOpParam.Op.CREATE;
1139 return new FsPathOutputStreamRunner(op, f, bufferSize,
1140 new PermissionParam(applyUMask(permission)),
1141 new OverwriteParam(overwrite),
1142 new BufferSizeParam(bufferSize),
1143 new ReplicationParam(replication),
1144 new BlockSizeParam(blockSize)
1145 ).run();
1146 }
1147
1148 @Override
1149 public FSDataOutputStream append(final Path f, final int bufferSize,
1150 final Progressable progress) throws IOException {
1151 statistics.incrementWriteOps(1);
1152
1153 final HttpOpParam.Op op = PostOpParam.Op.APPEND;
1154 return new FsPathOutputStreamRunner(op, f, bufferSize,
1155 new BufferSizeParam(bufferSize)
1156 ).run();
1157 }
1158
1159 @Override
1160 public boolean delete(Path f, boolean recursive) throws IOException {
1161 final HttpOpParam.Op op = DeleteOpParam.Op.DELETE;
1162 return new FsPathBooleanRunner(op, f,
1163 new RecursiveParam(recursive)
1164 ).run();
1165 }
1166
1167 @Override
1168 public FSDataInputStream open(final Path f, final int buffersize
1169 ) throws IOException {
1170 statistics.incrementReadOps(1);
1171 final HttpOpParam.Op op = GetOpParam.Op.OPEN;
1172 // use a runner so the open can recover from an invalid token
1173 FsPathConnectionRunner runner =
1174 new FsPathConnectionRunner(op, f, new BufferSizeParam(buffersize));
1175 return new FSDataInputStream(new OffsetUrlInputStream(
1176 new UnresolvedUrlOpener(runner), new OffsetUrlOpener(null)));
1177 }
1178
1179 @Override
1180 public synchronized void close() throws IOException {
1181 try {
1182 if (canRefreshDelegationToken && delegationToken != null) {
1183 cancelDelegationToken(delegationToken);
1184 }
1185 } catch (IOException ioe) {
1186 LOG.debug("Token cancel failed: "+ioe);
1187 } finally {
1188 super.close();
1189 }
1190 }
1191
1192 // use FsPathConnectionRunner to ensure retries for InvalidTokens
1193 class UnresolvedUrlOpener extends ByteRangeInputStream.URLOpener {
1194 private final FsPathConnectionRunner runner;
1195 UnresolvedUrlOpener(FsPathConnectionRunner runner) {
1196 super(null);
1197 this.runner = runner;
1198 }
1199
1200 @Override
1201 protected HttpURLConnection connect(long offset, boolean resolved)
1202 throws IOException {
1203 assert offset == 0;
1204 HttpURLConnection conn = runner.run();
1205 setURL(conn.getURL());
1206 return conn;
1207 }
1208 }
1209
1210 class OffsetUrlOpener extends ByteRangeInputStream.URLOpener {
1211 OffsetUrlOpener(final URL url) {
1212 super(url);
1213 }
1214
1215 /** Setup offset url and connect. */
1216 @Override
1217 protected HttpURLConnection connect(final long offset,
1218 final boolean resolved) throws IOException {
1219 final URL offsetUrl = offset == 0L? url
1220 : new URL(url + "&" + new OffsetParam(offset));
1221 return new URLRunner(GetOpParam.Op.OPEN, offsetUrl, resolved).run();
1222 }
1223 }
1224
1225 private static final String OFFSET_PARAM_PREFIX = OffsetParam.NAME + "=";
1226
1227 /** Remove offset parameter, if there is any, from the url */
1228 static URL removeOffsetParam(final URL url) throws MalformedURLException {
1229 String query = url.getQuery();
1230 if (query == null) {
1231 return url;
1232 }
1233 final String lower = query.toLowerCase();
1234 if (!lower.startsWith(OFFSET_PARAM_PREFIX)
1235 && !lower.contains("&" + OFFSET_PARAM_PREFIX)) {
1236 return url;
1237 }
1238
1239 //rebuild query
1240 StringBuilder b = null;
1241 for(final StringTokenizer st = new StringTokenizer(query, "&");
1242 st.hasMoreTokens();) {
1243 final String token = st.nextToken();
1244 if (!token.toLowerCase().startsWith(OFFSET_PARAM_PREFIX)) {
1245 if (b == null) {
1246 b = new StringBuilder("?").append(token);
1247 } else {
1248 b.append('&').append(token);
1249 }
1250 }
1251 }
1252 query = b == null? "": b.toString();
1253
1254 final String urlStr = url.toString();
1255 return new URL(urlStr.substring(0, urlStr.indexOf('?')) + query);
1256 }
1257
1258 static class OffsetUrlInputStream extends ByteRangeInputStream {
1259 OffsetUrlInputStream(UnresolvedUrlOpener o, OffsetUrlOpener r)
1260 throws IOException {
1261 super(o, r);
1262 }
1263
1264 /** Remove offset parameter before returning the resolved url. */
1265 @Override
1266 protected URL getResolvedUrl(final HttpURLConnection connection
1267 ) throws MalformedURLException {
1268 return removeOffsetParam(connection.getURL());
1269 }
1270 }
1271
1272 @Override
1273 public FileStatus[] listStatus(final Path f) throws IOException {
1274 statistics.incrementReadOps(1);
1275
1276 final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
1277 return new FsPathResponseRunner<FileStatus[]>(op, f) {
1278 @Override
1279 FileStatus[] decodeResponse(Map<?,?> json) {
1280 final Map<?, ?> rootmap = (Map<?, ?>)json.get(FileStatus.class.getSimpleName() + "es");
1281 final Object[] array = (Object[])rootmap.get(FileStatus.class.getSimpleName());
1282
1283 //convert FileStatus
1284 final FileStatus[] statuses = new FileStatus[array.length];
1285 for (int i = 0; i < array.length; i++) {
1286 final Map<?, ?> m = (Map<?, ?>)array[i];
1287 statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
1288 }
1289 return statuses;
1290 }
1291 }.run();
1292 }
1293
1294 @Override
1295 public Token<DelegationTokenIdentifier> getDelegationToken(
1296 final String renewer) throws IOException {
1297 final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
1298 Token<DelegationTokenIdentifier> token =
1299 new FsPathResponseRunner<Token<DelegationTokenIdentifier>>(
1300 op, null, new RenewerParam(renewer)) {
1301 @Override
1302 Token<DelegationTokenIdentifier> decodeResponse(Map<?,?> json)
1303 throws IOException {
1304 return JsonUtil.toDelegationToken(json);
1305 }
1306 }.run();
1307 if (token != null) {
1308 token.setService(tokenServiceName);
1309 } else {
1310 if (disallowFallbackToInsecureCluster) {
1311 throw new AccessControlException(CANT_FALLBACK_TO_INSECURE_MSG);
1312 }
1313 }
1314 return token;
1315 }
1316
1317 @Override
1318 public synchronized Token<?> getRenewToken() {
1319 return delegationToken;
1320 }
1321
1322 @Override
1323 public <T extends TokenIdentifier> void setDelegationToken(
1324 final Token<T> token) {
1325 synchronized (this) {
1326 delegationToken = token;
1327 }
1328 }
1329
1330 @Override
1331 public synchronized long renewDelegationToken(final Token<?> token
1332 ) throws IOException {
1333 final HttpOpParam.Op op = PutOpParam.Op.RENEWDELEGATIONTOKEN;
1334 return new FsPathResponseRunner<Long>(op, null,
1335 new TokenArgumentParam(token.encodeToUrlString())) {
1336 @Override
1337 Long decodeResponse(Map<?,?> json) throws IOException {
1338 return (Long) json.get("long");
1339 }
1340 }.run();
1341 }
1342
1343 @Override
1344 public synchronized void cancelDelegationToken(final Token<?> token
1345 ) throws IOException {
1346 final HttpOpParam.Op op = PutOpParam.Op.CANCELDELEGATIONTOKEN;
1347 new FsPathRunner(op, null,
1348 new TokenArgumentParam(token.encodeToUrlString())
1349 ).run();
1350 }
1351
1352 @Override
1353 public BlockLocation[] getFileBlockLocations(final FileStatus status,
1354 final long offset, final long length) throws IOException {
1355 if (status == null) {
1356 return null;
1357 }
1358 return getFileBlockLocations(status.getPath(), offset, length);
1359 }
1360
1361 @Override
1362 public BlockLocation[] getFileBlockLocations(final Path p,
1363 final long offset, final long length) throws IOException {
1364 statistics.incrementReadOps(1);
1365
1366 final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS;
1367 return new FsPathResponseRunner<BlockLocation[]>(op, p,
1368 new OffsetParam(offset), new LengthParam(length)) {
1369 @Override
1370 BlockLocation[] decodeResponse(Map<?,?> json) throws IOException {
1371 return DFSUtil.locatedBlocks2Locations(
1372 JsonUtil.toLocatedBlocks(json));
1373 }
1374 }.run();
1375 }
1376
1377 @Override
1378 public void access(final Path path, final FsAction mode) throws IOException {
1379 final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;
1380 new FsPathRunner(op, path, new FsActionParam(mode)).run();
1381 }
1382
1383 @Override
1384 public ContentSummary getContentSummary(final Path p) throws IOException {
1385 statistics.incrementReadOps(1);
1386
1387 final HttpOpParam.Op op = GetOpParam.Op.GETCONTENTSUMMARY;
1388 return new FsPathResponseRunner<ContentSummary>(op, p) {
1389 @Override
1390 ContentSummary decodeResponse(Map<?,?> json) {
1391 return JsonUtil.toContentSummary(json);
1392 }
1393 }.run();
1394 }
1395
1396 @Override
1397 public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
1398 ) throws IOException {
1399 statistics.incrementReadOps(1);
1400
1401 final HttpOpParam.Op op = GetOpParam.Op.GETFILECHECKSUM;
1402 return new FsPathResponseRunner<MD5MD5CRC32FileChecksum>(op, p) {
1403 @Override
1404 MD5MD5CRC32FileChecksum decodeResponse(Map<?,?> json) throws IOException {
1405 return JsonUtil.toMD5MD5CRC32FileChecksum(json);
1406 }
1407 }.run();
1408 }
1409
1410 /**
1411 * Resolve an HDFS URL into real INetSocketAddress. It works like a DNS
1412 * resolver when the URL points to an non-HA cluster. When the URL points to
1413 * an HA cluster with its logical name, the resolver further resolves the
1414 * logical name(i.e., the authority in the URL) into real namenode addresses.
1415 */
1416 private InetSocketAddress[] resolveNNAddr() throws IOException {
1417 Configuration conf = getConf();
1418 final String scheme = uri.getScheme();
1419
1420 ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>();
1421
1422 if (!HAUtil.isLogicalUri(conf, uri)) {
1423 InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(),
1424 getDefaultPort());
1425 ret.add(addr);
1426
1427 } else {
1428 Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil
1429 .getHaNnWebHdfsAddresses(conf, scheme);
1430
1431 // Extract the entry corresponding to the logical name.
1432 Map<String, InetSocketAddress> addrs = addresses.get(uri.getHost());
1433 for (InetSocketAddress addr : addrs.values()) {
1434 ret.add(addr);
1435 }
1436 }
1437
1438 InetSocketAddress[] r = new InetSocketAddress[ret.size()];
1439 return ret.toArray(r);
1440 }
1441
1442 @Override
1443 public String getCanonicalServiceName() {
1444 return tokenServiceName == null ? super.getCanonicalServiceName()
1445 : tokenServiceName.toString();
1446 }
1447
1448 @VisibleForTesting
1449 InetSocketAddress[] getResolvedNNAddr() {
1450 return nnAddrs;
1451 }
1452 }