Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.datanode.web.resources;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.net.InetSocketAddress;
023import java.net.URI;
024import java.net.URISyntaxException;
025import java.security.PrivilegedExceptionAction;
026import java.util.EnumSet;
027
028import javax.servlet.ServletContext;
029import javax.servlet.http.HttpServletResponse;
030import javax.ws.rs.Consumes;
031import javax.ws.rs.DefaultValue;
032import javax.ws.rs.GET;
033import javax.ws.rs.POST;
034import javax.ws.rs.PUT;
035import javax.ws.rs.Path;
036import javax.ws.rs.PathParam;
037import javax.ws.rs.Produces;
038import javax.ws.rs.QueryParam;
039import javax.ws.rs.core.Context;
040import javax.ws.rs.core.MediaType;
041import javax.ws.rs.core.Response;
042
043import org.apache.commons.logging.Log;
044import org.apache.commons.logging.LogFactory;
045import org.apache.hadoop.conf.Configuration;
046import org.apache.hadoop.fs.CreateFlag;
047import org.apache.hadoop.fs.FSDataOutputStream;
048import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
049import org.apache.hadoop.fs.permission.FsPermission;
050import org.apache.hadoop.hdfs.DFSClient;
051import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
052import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
053import org.apache.hadoop.hdfs.server.datanode.DataNode;
054import org.apache.hadoop.hdfs.server.namenode.NameNode;
055import org.apache.hadoop.hdfs.web.JsonUtil;
056import org.apache.hadoop.hdfs.web.ParamFilter;
057import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
058import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
059import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
060import org.apache.hadoop.hdfs.web.resources.DelegationParam;
061import org.apache.hadoop.hdfs.web.resources.GetOpParam;
062import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
063import org.apache.hadoop.hdfs.web.resources.LengthParam;
064import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam;
065import org.apache.hadoop.hdfs.web.resources.OffsetParam;
066import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
067import org.apache.hadoop.hdfs.web.resources.Param;
068import org.apache.hadoop.hdfs.web.resources.PermissionParam;
069import org.apache.hadoop.hdfs.web.resources.PostOpParam;
070import org.apache.hadoop.hdfs.web.resources.PutOpParam;
071import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
072import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
073import org.apache.hadoop.io.IOUtils;
074import org.apache.hadoop.security.SecurityUtil;
075import org.apache.hadoop.security.UserGroupInformation;
076import org.apache.hadoop.security.token.Token;
077
078import com.sun.jersey.spi.container.ResourceFilters;
079
080/** Web-hdfs DataNode implementation. */
081@Path("")
082@ResourceFilters(ParamFilter.class)
083public class DatanodeWebHdfsMethods {
084  public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
085
086  private static final UriFsPathParam ROOT = new UriFsPathParam("");
087
088  private @Context ServletContext context;
089  private @Context HttpServletResponse response;
090
091  private void init(final UserGroupInformation ugi,
092      final DelegationParam delegation, final InetSocketAddress nnRpcAddr,
093      final UriFsPathParam path, final HttpOpParam<?> op,
094      final Param<?, ?>... parameters) throws IOException {
095    if (LOG.isTraceEnabled()) {
096      LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
097          + ", ugi=" + ugi + Param.toSortedString(", ", parameters));
098    }
099
100    //clear content type
101    response.setContentType(null);
102    
103    if (UserGroupInformation.isSecurityEnabled()) {
104      //add a token for RPC.
105      final Token<DelegationTokenIdentifier> token = 
106          new Token<DelegationTokenIdentifier>();
107      token.decodeFromUrlString(delegation.getValue());
108      SecurityUtil.setTokenService(token, nnRpcAddr);
109      token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
110      ugi.addToken(token);
111    }
112  }
113
114  /** Handle HTTP PUT request for the root. */
115  @PUT
116  @Path("/")
117  @Consumes({"*/*"})
118  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
119  public Response putRoot(
120      final InputStream in,
121      @Context final UserGroupInformation ugi,
122      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
123          final DelegationParam delegation,
124      @QueryParam(NamenodeRpcAddressParam.NAME) 
125      @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 
126          final NamenodeRpcAddressParam namenodeRpcAddress,
127      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
128          final PutOpParam op,
129      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
130          final PermissionParam permission,
131      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
132          final OverwriteParam overwrite,
133      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
134          final BufferSizeParam bufferSize,
135      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
136          final ReplicationParam replication,
137      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
138          final BlockSizeParam blockSize
139      ) throws IOException, InterruptedException {
140    return put(in, ugi, delegation, namenodeRpcAddress, ROOT, op, permission,
141        overwrite, bufferSize, replication, blockSize);
142  }
143
144  /** Handle HTTP PUT request. */
145  @PUT
146  @Path("{" + UriFsPathParam.NAME + ":.*}")
147  @Consumes({"*/*"})
148  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
149  public Response put(
150      final InputStream in,
151      @Context final UserGroupInformation ugi,
152      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
153          final DelegationParam delegation,
154      @QueryParam(NamenodeRpcAddressParam.NAME)
155      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
156          final NamenodeRpcAddressParam namenodeRpcAddress,
157      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
158      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
159          final PutOpParam op,
160      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
161          final PermissionParam permission,
162      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
163          final OverwriteParam overwrite,
164      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
165          final BufferSizeParam bufferSize,
166      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
167          final ReplicationParam replication,
168      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
169          final BlockSizeParam blockSize
170      ) throws IOException, InterruptedException {
171
172    final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
173    init(ugi, delegation, nnRpcAddr, path, op, permission,
174        overwrite, bufferSize, replication, blockSize);
175
176    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
177      @Override
178      public Response run() throws IOException, URISyntaxException {
179
180    final String fullpath = path.getAbsolutePath();
181    final DataNode datanode = (DataNode)context.getAttribute("datanode");
182
183    switch(op.getValue()) {
184    case CREATE:
185    {
186      final Configuration conf = new Configuration(datanode.getConf());
187      conf.set(FsPermission.UMASK_LABEL, "000");
188
189      final int b = bufferSize.getValue(conf);
190      DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
191      FSDataOutputStream out = null;
192      try {
193        out = new FSDataOutputStream(dfsclient.create(
194            fullpath, permission.getFsPermission(), 
195            overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
196                : EnumSet.of(CreateFlag.CREATE),
197            replication.getValue(conf), blockSize.getValue(conf), null, b, null), null);
198        IOUtils.copyBytes(in, out, b);
199        out.close();
200        out = null;
201        dfsclient.close();
202        dfsclient = null;
203      } finally {
204        IOUtils.cleanup(LOG, out);
205        IOUtils.cleanup(LOG, dfsclient);
206      }
207      final InetSocketAddress nnHttpAddr = NameNode.getHttpAddress(conf);
208      final URI uri = new URI(WebHdfsFileSystem.SCHEME, null,
209          nnHttpAddr.getHostName(), nnHttpAddr.getPort(), fullpath, null, null);
210      return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
211    }
212    default:
213      throw new UnsupportedOperationException(op + " is not supported");
214    }
215      }
216    });
217  }
218
219  /** Handle HTTP POST request for the root for the root. */
220  @POST
221  @Path("/")
222  @Consumes({"*/*"})
223  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
224  public Response postRoot(
225      final InputStream in,
226      @Context final UserGroupInformation ugi,
227      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
228          final DelegationParam delegation,
229      @QueryParam(NamenodeRpcAddressParam.NAME)
230      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
231          final NamenodeRpcAddressParam namenodeRpcAddress,
232      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
233          final PostOpParam op,
234      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
235          final BufferSizeParam bufferSize
236      ) throws IOException, InterruptedException {
237    return post(in, ugi, delegation, namenodeRpcAddress, ROOT, op, bufferSize);
238  }
239
240  /** Handle HTTP POST request. */
241  @POST
242  @Path("{" + UriFsPathParam.NAME + ":.*}")
243  @Consumes({"*/*"})
244  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
245  public Response post(
246      final InputStream in,
247      @Context final UserGroupInformation ugi,
248      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
249          final DelegationParam delegation,
250      @QueryParam(NamenodeRpcAddressParam.NAME)
251      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
252          final NamenodeRpcAddressParam namenodeRpcAddress,
253      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
254      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
255          final PostOpParam op,
256      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
257          final BufferSizeParam bufferSize
258      ) throws IOException, InterruptedException {
259
260    final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
261    init(ugi, delegation, nnRpcAddr, path, op, bufferSize);
262
263    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
264      @Override
265      public Response run() throws IOException {
266
267    final String fullpath = path.getAbsolutePath();
268    final DataNode datanode = (DataNode)context.getAttribute("datanode");
269
270    switch(op.getValue()) {
271    case APPEND:
272    {
273      final Configuration conf = new Configuration(datanode.getConf());
274      final int b = bufferSize.getValue(conf);
275      DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
276      FSDataOutputStream out = null;
277      try {
278        out = dfsclient.append(fullpath, b, null, null);
279        IOUtils.copyBytes(in, out, b);
280        out.close();
281        out = null;
282        dfsclient.close();
283        dfsclient = null;
284      } finally {
285        IOUtils.cleanup(LOG, out);
286        IOUtils.cleanup(LOG, dfsclient);
287      }
288      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
289    }
290    default:
291      throw new UnsupportedOperationException(op + " is not supported");
292    }
293      }
294    });
295  }
296
297  /** Handle HTTP GET request for the root. */
298  @GET
299  @Path("/")
300  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
301  public Response getRoot(
302      @Context final UserGroupInformation ugi,
303      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
304          final DelegationParam delegation,
305      @QueryParam(NamenodeRpcAddressParam.NAME)
306      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
307          final NamenodeRpcAddressParam namenodeRpcAddress,
308      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
309          final GetOpParam op,
310      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
311          final OffsetParam offset,
312      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
313          final LengthParam length,
314      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
315          final BufferSizeParam bufferSize
316      ) throws IOException, InterruptedException {
317    return get(ugi, delegation, namenodeRpcAddress, ROOT, op, offset, length,
318        bufferSize);
319  }
320
321  /** Handle HTTP GET request. */
322  @GET
323  @Path("{" + UriFsPathParam.NAME + ":.*}")
324  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
325  public Response get(
326      @Context final UserGroupInformation ugi,
327      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
328          final DelegationParam delegation,
329      @QueryParam(NamenodeRpcAddressParam.NAME)
330      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
331          final NamenodeRpcAddressParam namenodeRpcAddress,
332      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
333      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
334          final GetOpParam op,
335      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
336          final OffsetParam offset,
337      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
338          final LengthParam length,
339      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
340          final BufferSizeParam bufferSize
341      ) throws IOException, InterruptedException {
342
343    final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
344    init(ugi, delegation, nnRpcAddr, path, op, offset, length, bufferSize);
345
346    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
347      @Override
348      public Response run() throws IOException {
349
350    final String fullpath = path.getAbsolutePath();
351    final DataNode datanode = (DataNode)context.getAttribute("datanode");
352    final Configuration conf = new Configuration(datanode.getConf());
353
354    switch(op.getValue()) {
355    case OPEN:
356    {
357      final int b = bufferSize.getValue(conf);
358      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
359      DFSDataInputStream in = null;
360      try {
361        in = new DFSClient.DFSDataInputStream(
362            dfsclient.open(fullpath, b, true));
363        in.seek(offset.getValue());
364      } catch(IOException ioe) {
365        IOUtils.cleanup(LOG, in);
366        IOUtils.cleanup(LOG, dfsclient);
367        throw ioe;
368      }
369      
370      final long n = length.getValue() != null? length.getValue()
371        : in.getVisibleLength() - offset.getValue();
372      return Response.ok(new OpenEntity(in, n, dfsclient)).type(
373          MediaType.APPLICATION_OCTET_STREAM).build();
374    }
375    case GETFILECHECKSUM:
376    {
377      MD5MD5CRC32FileChecksum checksum = null;
378      DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
379      try {
380        checksum = dfsclient.getFileChecksum(fullpath);
381        dfsclient.close();
382        dfsclient = null;
383      } finally {
384        IOUtils.cleanup(LOG, dfsclient);
385      }
386      final String js = JsonUtil.toJsonString(checksum);
387      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
388    }
389    default:
390      throw new UnsupportedOperationException(op + " is not supported");
391    }
392      }
393    });
394  }
395}