Grid Community Toolkit  6.2.1705709074 (tag: v6.2.20240202)
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
gridftp_hdfs.h
1 
2 /*
3  * Portions of this file Copyright 2008-2011 University of Nebraska-Lincoln
4  *
5  * This file is licensed under the
6  * terms of the Apache Public License, found at
7  * http://www.apache.org/licenses/LICENSE-2.0.html.
8  */
9 
10 
11 #include <hdfs.h>
12 #include <stdint.h>
13 #include <openssl/md5.h>
14 
15 #include "globus_gridftp_server.h"
16 #include "gridftp_hdfs_error.h"
17 
18 
19 #ifndef _GNU_SOURCE
20 #define _GNU_SOURCE
21 #endif
22 
23 // Note that we hide all symbols from the global scope except the module itself.
24 #pragma GCC visibility push(hidden)
25 
26 // Data types and globals
27 #define default_id 00;
28 
29 // Note: This really should be const, but the globus module activation code
30 // doesn't have this as const.
31 extern globus_version_t gridftp_hdfs_local_version;
32 
33 #define HDFS_CKSM_TYPE_CKSUM 1
34 #define HDFS_CKSM_TYPE_CRC32 2
35 #define HDFS_CKSM_TYPE_ADLER32 4
36 #define HDFS_CKSM_TYPE_MD5 8
37 
38 typedef struct globus_l_gfs_hdfs_handle_s
39 {
40  char * pathname;
41  hdfsFS fs;
42  hdfsFile fd;
43  globus_off_t file_size; // size of the file for reads
44  globus_size_t block_size;
45  globus_off_t op_length; // Length of the requested read/write size
46  globus_off_t offset;
47  unsigned int done;
48  globus_result_t done_status; // The status of the finished transfer.
49  globus_bool_t sent_finish; // Whether or not we have sent the client an abort.
50  globus_gfs_operation_t op;
51  globus_byte_t * buffer;
52  globus_off_t * offsets; // The offset of each buffer.
53  globus_size_t * nbytes; // The number of bytes in each buffer.
54  short * used;
55  int optimal_count;
56  unsigned int max_buffer_count;
57  unsigned int max_file_buffer_count;
58  unsigned int buffer_count; // Number of buffers we currently maintain in memory waiting to be written to HDFS.
59  unsigned int outstanding;
60  globus_mutex_t * mutex;
61  int port;
62  char * host;
63  char * mount_point;
64  unsigned int mount_point_len;
65  unsigned int replicas;
66  char * username;
67  char * tmp_file_pattern;
68  int tmpfilefd;
69  globus_bool_t using_file_buffer;
70  char * syslog_host; // The host to send syslog message to.
71  char * remote_host; // The remote host connecting to us.
72  char * local_host; // Our local hostname.
73  char * syslog_msg; // Message printed out to syslog.
74  unsigned int io_block_size;
75  unsigned long long io_count;
76  globus_bool_t eof;
77 
78  // Checksumming support
79  char * expected_cksm;
80  const char * cksm_root;
81  unsigned char cksm_types;
82  MD5_CTX md5;
83  char md5_output[MD5_DIGEST_LENGTH];
84  char md5_output_human[MD5_DIGEST_LENGTH*2+1];
85  uint32_t adler32;
86  char adler32_human[2*sizeof(uint32_t)+1];
87  uint32_t crc32;
88  uint32_t cksum;
89 } globus_l_gfs_hdfs_handle_t;
90 typedef globus_l_gfs_hdfs_handle_t hdfs_handle_t;
91 
92 #define MSG_SIZE 1024
93 extern char err_msg[MSG_SIZE];
94 
95 // Function for sending a file to the client.
96 void
97 hdfs_send(
98  globus_gfs_operation_t op,
99  globus_gfs_transfer_info_t * transfer_info,
100  void * user_arg);
101 
102 
103 // Function for receiving a file from the client.
104 void
105 hdfs_recv(
106  globus_gfs_operation_t op,
107  globus_gfs_transfer_info_t * transfer_info,
108  void * user_arg);
109 
110 // Buffer management for writes
112 hdfs_store_buffer(
113  globus_l_gfs_hdfs_handle_t * hdfs_handle,
114  globus_byte_t* buffer,
115  globus_off_t offset,
116  globus_size_t nbytes);
117 
119 hdfs_dump_buffers(
120  globus_l_gfs_hdfs_handle_t * hdfs_handle);
121 
123 hdfs_dump_buffer_immed(
124  hdfs_handle_t * hdfs_handle,
125  globus_byte_t * buffer,
126  globus_size_t nbytes);
127 
128 // Buffer management for reads
130 allocate_buffers(
131  hdfs_handle_t * hdfs_handle,
132  globus_size_t num_buffers);
133 
134 globus_ssize_t
135 find_buffer(
136  hdfs_handle_t * hdfs_handle,
137  globus_byte_t * buffer);
138 
139 globus_ssize_t
140 find_empty_buffer(
141  hdfs_handle_t * hdfs_handle);
142 
143 void
144 disgard_buffer(
145  hdfs_handle_t * hdfs_handle,
146  globus_ssize_t idx);
147 
148 void
149 remove_file_buffer(
150  hdfs_handle_t * hdfs_handle);
151 
152 
153 // Metadata-related functions
154 void
155 hdfs_stat(
156  globus_gfs_operation_t op,
157  globus_gfs_stat_info_t * stat_info,
158  void * user_arg);
159 
160 // Some helper functions
161 // All must be called with the hdfs_handle mutex held
162 void
163 set_done(
164  hdfs_handle_t * hdfs_handle,
165  globus_result_t rc);
166 
167 void
168 set_close_done(
169  hdfs_handle_t * hdfs_handle,
170  globus_result_t rc);
171 
173 is_done(
174  hdfs_handle_t * hdfs_handle);
175 
177 is_close_done(
178  hdfs_handle_t * hdfs_handle);
179 
180 // Checksumming support
181 void
182 hdfs_parse_checksum_types(
183  hdfs_handle_t * hdfs_handle,
184  const char * types);
185 
186 void
187 hdfs_initialize_checksums(
188  hdfs_handle_t * hdfs_handle);
189 
190 void
191 hdfs_update_checksums(
192  hdfs_handle_t * hdfs_handle,
193  globus_byte_t * buffer,
194  globus_size_t nbytes);
195 
196 void
197 hdfs_finalize_checksums(
198  hdfs_handle_t * hdfs_handle);
199 
201 hdfs_save_checksum(
202  hdfs_handle_t * hdfs_handle);
203 
205 hdfs_get_checksum(
206  hdfs_handle_t * hdfs_handle,
207  const char * pathname,
208  const char * requested_cksm,
209  char ** cksm_value);
210 
211 #pragma GCC visibility pop
212 
int globus_bool_t
Boolean type.
Definition: globus_types.h:93
Mutex.
Definition: globus_thread.h:107
size_t globus_size_t
Standard size of memory objectThe globus_size_t is the size of a memory object. It is identical to si...
Definition: globus_types.h:48
unsigned char globus_byte_t
Unsigned byte datatypeThis is used for byte-addressable arrays of arbitrary data which is not subject...
Definition: globus_types.h:85
uint32_t globus_result_t
Definition: globus_types.h:99