fork download
copy 
  1. # EXPERIMENT 7
  2. # HDFS File Management Operations
  3. # Online Compiler Simulation
  4.  
  5. print("========================================")
  6. print("     HDFS FILE MANAGEMENT SIMULATION")
  7. print("========================================")
  8.  
  9. # ----------------------------------------
  10. # Simulated HDFS
  11. # ----------------------------------------
  12.  
  13. hdfs = {
  14.     "/user/student/dir1": {},
  15.     "/user/student/dir2": {}
  16. }
  17.  
  18. # ----------------------------------------
  19. # 1. CREATE DIRECTORIES
  20. # ----------------------------------------
  21.  
  22. print("\n1. CREATE DIRECTORIES")
  23. print("----------------------------------------")
  24.  
  25. print("hadoop fs -mkdir /user/student/dir1")
  26. print("Directory created successfully")
  27.  
  28. print("hadoop fs -mkdir /user/student/dir2")
  29. print("Directory created successfully")
  30.  
  31.  
  32. # ----------------------------------------
  33. # 2. CREATE AND UPLOAD FILE
  34. # ----------------------------------------
  35.  
  36. print("\n2. UPLOAD FILE TO HDFS")
  37. print("----------------------------------------")
  38.  
  39. sample_content = """Big Data Analytics
  40. Hadoop is used for distributed processing.
  41. HDFS provides distributed storage.
  42. MapReduce processes large datasets.
  43. Big Data is useful for business analytics."""
  44.  
  45. hdfs["/user/student/dir1"]["sample.txt"] = sample_content
  46.  
  47. print("hadoop fs -put sample.txt /user/student/dir1/")
  48. print("File uploaded successfully")
  49.  
  50.  
  51. # ----------------------------------------
  52. # 3. LIST DIRECTORY
  53. # ----------------------------------------
  54.  
  55. print("\n3. LIST HDFS DIRECTORY")
  56. print("----------------------------------------")
  57.  
  58. print("hadoop fs -ls /user/student/dir1")
  59.  
  60. for filename in hdfs["/user/student/dir1"]:
  61.     print("sample.txt")
  62.  
  63.  
  64. # ----------------------------------------
  65. # 4. DISPLAY FILE CONTENT
  66. # ----------------------------------------
  67.  
  68. print("\n4. DISPLAY FILE CONTENT")
  69. print("----------------------------------------")
  70.  
  71. print("hadoop fs -cat /user/student/dir1/sample.txt")
  72. print()
  73.  
  74. print(hdfs["/user/student/dir1"]["sample.txt"])
  75.  
  76.  
  77. # ----------------------------------------
  78. # 5. COPY FILE
  79. # ----------------------------------------
  80.  
  81. print("\n5. COPY FILE")
  82. print("----------------------------------------")
  83.  
  84. content = hdfs["/user/student/dir1"]["sample.txt"]
  85.  
  86. hdfs["/user/student/dir2"]["sample_copy.txt"] = content
  87.  
  88. print("hadoop fs -cp /user/student/dir1/sample.txt /user/student/dir2/")
  89. print("File copied successfully")
  90.  
  91.  
  92. # ----------------------------------------
  93. # 6. MOVE FILE
  94. # ----------------------------------------
  95.  
  96. print("\n6. MOVE FILE")
  97. print("----------------------------------------")
  98.  
  99. hdfs["/user/student/dir2"]["moved.txt"] = "This file has been moved."
  100.  
  101. print("hadoop fs -mv /user/student/dir1/moved.txt /user/student/dir2/")
  102. print("File moved successfully")
  103.  
  104.  
  105. # ----------------------------------------
  106. # 7. DOWNLOAD FILE
  107. # ----------------------------------------
  108.  
  109. print("\n7. DOWNLOAD FILE")
  110. print("----------------------------------------")
  111.  
  112. downloaded_file = hdfs["/user/student/dir2"]["sample_copy.txt"]
  113.  
  114. print("hadoop fs -get /user/student/dir2/sample_copy.txt")
  115. print("File downloaded successfully")
  116. print("Downloaded content:")
  117. print(downloaded_file)
  118.  
  119.  
  120. # ----------------------------------------
  121. # 8. DISPLAY LAST FEW LINES
  122. # ----------------------------------------
  123.  
  124. print("\n8. DISPLAY LAST FEW LINES")
  125. print("----------------------------------------")
  126.  
  127. print("hadoop fs -tail /user/student/dir1/sample.txt")
  128. print()
  129.  
  130. lines = hdfs["/user/student/dir1"]["sample.txt"].split("\n")
  131.  
  132. for line in lines[-3:]:
  133.     print(line)
  134.  
  135.  
  136. # ----------------------------------------
  137. # 9. DISPLAY FILE SIZE
  138. # ----------------------------------------
  139.  
  140. print("\n9. DISPLAY FILE SIZE")
  141. print("----------------------------------------")
  142.  
  143. file_content = hdfs["/user/student/dir1"]["sample.txt"]
  144.  
  145. file_size = len(file_content.encode("utf-8"))
  146.  
  147. print("hadoop fs -du /user/student/dir1/sample.txt")
  148. print("File size:", file_size, "bytes")
  149.  
  150.  
  151. # ----------------------------------------
  152. # 10. DELETE FILE
  153. # ----------------------------------------
  154.  
  155. print("\n10. DELETE FILE")
  156. print("----------------------------------------")
  157.  
  158. del hdfs["/user/student/dir2"]["moved.txt"]
  159.  
  160. print("hadoop fs -rm /user/student/dir2/moved.txt")
  161. print("File deleted successfully")
  162.  
  163.  
  164. # ----------------------------------------
  165. # 11. FINAL DIRECTORY CONTENT
  166. # ----------------------------------------
  167.  
  168. print("\n11. FINAL HDFS CONTENT")
  169. print("----------------------------------------")
  170.  
  171. print("/user/student/dir1")
  172.  
  173. for filename in hdfs["/user/student/dir1"]:
  174.     print("   ", filename)
  175.  
  176. print("\n/user/student/dir2")
  177.  
  178. for filename in hdfs["/user/student/dir2"]:
  179.     print("   ", filename)
  180.  
  181.  
  182. # ----------------------------------------
  183. # RESULT
  184. # ----------------------------------------
  185.  
  186. print("\n========================================")
  187. print("                 RESULT")
  188. print("========================================")
  189.  
  190. print("Directory creation  : SUCCESS")
  191. print("File upload         : SUCCESS")
  192. print("Directory listing   : SUCCESS")
  193. print("File retrieval      : SUCCESS")
  194. print("File copying        : SUCCESS")
  195. print("File moving         : SUCCESS")
  196. print("File downloading    : SUCCESS")
  197. print("File deletion       : SUCCESS")
  198. print("Tail operation      : SUCCESS")
  199. print("File size operation : SUCCESS")
  200.  
  201. print("\nExperiment 7 completed successfully.")
Success #stdin #stdout 0.01s 7108KB
comments (?)
stdin
copy 
Standard input is empty
stdout
copy 
========================================
     HDFS FILE MANAGEMENT SIMULATION
========================================

1. CREATE DIRECTORIES
----------------------------------------
hadoop fs -mkdir /user/student/dir1
Directory created successfully
hadoop fs -mkdir /user/student/dir2
Directory created successfully

2. UPLOAD FILE TO HDFS
----------------------------------------
hadoop fs -put sample.txt /user/student/dir1/
File uploaded successfully

3. LIST HDFS DIRECTORY
----------------------------------------
hadoop fs -ls /user/student/dir1
sample.txt

4. DISPLAY FILE CONTENT
----------------------------------------
hadoop fs -cat /user/student/dir1/sample.txt
()
Big Data Analytics
Hadoop is used for distributed processing.
HDFS provides distributed storage.
MapReduce processes large datasets.
Big Data is useful for business analytics.

5. COPY FILE
----------------------------------------
hadoop fs -cp /user/student/dir1/sample.txt /user/student/dir2/
File copied successfully

6. MOVE FILE
----------------------------------------
hadoop fs -mv /user/student/dir1/moved.txt /user/student/dir2/
File moved successfully

7. DOWNLOAD FILE
----------------------------------------
hadoop fs -get /user/student/dir2/sample_copy.txt
File downloaded successfully
Downloaded content:
Big Data Analytics
Hadoop is used for distributed processing.
HDFS provides distributed storage.
MapReduce processes large datasets.
Big Data is useful for business analytics.

8. DISPLAY LAST FEW LINES
----------------------------------------
hadoop fs -tail /user/student/dir1/sample.txt
()
HDFS provides distributed storage.
MapReduce processes large datasets.
Big Data is useful for business analytics.

9. DISPLAY FILE SIZE
----------------------------------------
hadoop fs -du /user/student/dir1/sample.txt
('File size:', 175, 'bytes')

10. DELETE FILE
----------------------------------------
hadoop fs -rm /user/student/dir2/moved.txt
File deleted successfully

11. FINAL HDFS CONTENT
----------------------------------------
/user/student/dir1
('   ', 'sample.txt')

/user/student/dir2
('   ', 'sample_copy.txt')

========================================
                 RESULT
========================================
Directory creation  : SUCCESS
File upload         : SUCCESS
Directory listing   : SUCCESS
File retrieval      : SUCCESS
File copying        : SUCCESS
File moving         : SUCCESS
File downloading    : SUCCESS
File deletion       : SUCCESS
Tail operation      : SUCCESS
File size operation : SUCCESS

Experiment 7 completed successfully.
https://ideone.com/hcwJOh
language:
Python (cpython 2.7.16)
created:
1 day ago
visibility:
public

Share or Embed source code

Discover > Sphere Engine API

The brand new service which powers Ideone!

Discover > IDE Widget

Widget for compiling and running the source code in a web browser!