CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!

1
# -*- coding: binary -*-
2
module Rex
3
  module Parser
4
    ###
5
    #
6
    # This class parses the contents of an NTFS partition file.
7
    # Author : Danil Bazin <danil.bazin[at]hsc.fr> @danilbaz
8
    #
9
    ###
10
    class NTFS
11
      #
12
      # Initialize the NTFS class with an already open file handler
13
      #
14
      DATA_ATTRIBUTE_ID = 128
15
      INDEX_ROOT_ID = 144
16
      INDEX_ALLOCATION_ID = 160
17
      def initialize(file_handler)
18
        @file_handler = file_handler
19
        data = @file_handler.read(4096)
20
        # Boot sector reading
21
        @bytes_per_sector = data[11, 2].unpack('v')[0]
22
        @sector_per_cluster = data[13].unpack('C')[0]
23
        @cluster_per_mft_record = data[64].unpack('c')[0]
24
        if @cluster_per_mft_record < 0
25
          @bytes_per_mft_record = 2**(-@cluster_per_mft_record)
26
          @cluster_per_mft_record = @bytes_per_mft_record.to_f / @bytes_per_sector / @sector_per_cluster
27
        else
28
          @bytes_per_mft_record = @bytes_per_sector * @sector_per_cluster * @cluster_per_mft_record
29
        end
30
        @bytes_per_cluster = @sector_per_cluster * @bytes_per_sector
31
        @mft_logical_cluster_number = data[48, 8].unpack('Q<')[0]
32
        @mft_offset = @mft_logical_cluster_number * @sector_per_cluster * @bytes_per_sector
33
        @file_handler.seek(@mft_offset)
34
        @mft = @file_handler.read(@bytes_per_mft_record)
35
      end
36

37
      #
38
      # Gather the MFT entry corresponding to his number
39
      #
40
      def mft_record_from_mft_num(mft_num)
41
        mft_num_offset = mft_num * @cluster_per_mft_record
42
        mft_data_attribute = mft_record_attribute(@mft)[DATA_ATTRIBUTE_ID]['data']
43
        cluster_from_attribute_non_resident(mft_data_attribute, mft_num_offset, @bytes_per_mft_record)
44
      end
45

46
      #
47
      # Get the size of the file in the $FILENAME (64) attribute
48
      #
49
      def real_size_from_filenameattribute(attribute)
50
        filename_attribute = attribute
51
        filename_attribute[48, 8].unpack('Q<')[0]
52
      end
53

54
      #
55
      # Gather the name of the file from the $FILENAME (64) attribute
56
      #
57
      def filename_from_filenameattribute(attribute)
58
        filename_attribute = attribute
59
        length_of_name = filename_attribute[64].ord
60
        # uft16 *2
61
        d = ::Encoding::Converter.new('UTF-16LE', 'UTF-8')
62
        d.convert(filename_attribute[66, (length_of_name * 2)])
63
      end
64

65
      #
66
      # Get the file from the MFT number
67
      # The size must be given because the $FILENAME attribute
68
      # in the MFT entry does not contain it
69
      # The file is in $DATA (128) Attribute
70
      #
71
      def file_content_from_mft_num(mft_num, size)
72
        mft_record = mft_record_from_mft_num(mft_num)
73
        attribute_list = mft_record_attribute(mft_record)
74
        if attribute_list[DATA_ATTRIBUTE_ID]['resident']
75
          return attribute_list[DATA_ATTRIBUTE_ID]['data']
76
        else
77
          data_attribute = attribute_list[DATA_ATTRIBUTE_ID]['data']
78
          return cluster_from_attribute_non_resident(data_attribute)[0, size]
79
        end
80
      end
81

82
      #
83
      # parse one index record and return the name, MFT number and size of the file
84
      #
85
      def parse_index(index_entry)
86
        res = {}
87
        filename_size = index_entry[10, 2].unpack('v')[0]
88
        filename_attribute = index_entry[16, filename_size]
89
        # Should be 8 bytes but it doesn't work
90
        # mft_offset =  index_entry[0.unpack('Q<',:8])[0]
91
        # work with 4 bytes
92
        mft_offset =  index_entry[0, 4].unpack('V')[0]
93
        res[filename_from_filenameattribute(filename_attribute)] = {
94
          'mft_offset' => mft_offset,
95
          'file_size' => real_size_from_filenameattribute(filename_attribute) }
96
        res
97
      end
98

99
      #
100
      # parse index_record in $INDEX_ROOT and recursively index_record in
101
      # INDEX_ALLOCATION
102
      #
103
      def parse_index_list(index_record, index_allocation_attribute)
104
        offset_index_entry_list = index_record[0, 4].unpack('V')[0]
105
        index_size =  index_record[offset_index_entry_list + 8, 2].unpack('v')[0]
106
        index_size_in_bytes = index_size * @bytes_per_cluster
107
        index_entry = index_record[offset_index_entry_list, index_size]
108
        res = {}
109
        while index_entry[12, 4].unpack('V')[0] & 2 != 2
110
          res.update(parse_index(index_entry))
111
          # if son
112
          if index_entry[12, 4].unpack('V')[0] & 1 == 1
113
            # should be 8 bytes length
114
            vcn =  index_entry[-8, 4].unpack('V')[0]
115
            vcn_in_bytes = vcn * @bytes_per_cluster
116
            res_son = parse_index_list(index_allocation_attribute[vcn_in_bytes + 24, index_size_in_bytes], index_allocation_attribute)
117
            res.update(res_son)
118
          end
119
          offset_index_entry_list += index_size
120
          index_size =  index_record[offset_index_entry_list + 8, 2].unpack('v')[0]
121
          index_size_in_bytes = index_size * @bytes_per_cluster
122
          index_entry = index_record [offset_index_entry_list, index_size]
123
        end
124
        # if son on the last
125
        if index_entry[12, 4].unpack('V')[0] & 1 == 1
126
          # should be 8 bytes length
127
          vcn =  index_entry[-8, 4].unpack('V')[0]
128
          vcn_in_bytes = vcn * @bytes_per_cluster
129
          res_son = parse_index_list(index_allocation_attribute[vcn_in_bytes + 24, index_size_in_bytes], index_allocation_attribute)
130
          res.update(res_son)
131
        end
132
        res
133
      end
134

135
      #
136
      # return the list of files in attribute directory and their MFT number and size
137
      #
138
      def index_list_from_attributes(attributes)
139
        index_root_attribute = attributes[INDEX_ROOT_ID]
140
        index_record = index_root_attribute[16, index_root_attribute.length - 16]
141
        if attributes.key?(INDEX_ALLOCATION_ID)
142
          return parse_index_list(index_record, attributes[INDEX_ALLOCATION_ID])
143
        else
144
          return parse_index_list(index_record, '')
145
        end
146
      end
147

148
      def cluster_from_attribute_non_resident(attribute, cluster_num = 0, size_max = ((2**31) - 1))
149
        lowvcn = attribute[16, 8].unpack('Q<')[0]
150
        highvcn = attribute[24, 8].unpack('Q<')[0]
151
        offset = attribute[32, 2].unpack('v')[0]
152
        real_size = attribute[48, 8].unpack('Q<')[0]
153
        attribut = ''
154
        run_list_num = lowvcn
155
        old_offset = 0
156
        while run_list_num <= highvcn
157
          first_runlist_byte = attribute[offset].ord
158
          run_offset_size = first_runlist_byte >> 4
159
          run_length_size = first_runlist_byte & 15
160
          run_length = attribute[offset + 1, run_length_size]
161
          run_length += "\x00" * (8 - run_length_size)
162
          run_length = run_length.unpack('Q<')[0]
163

164
          offset_run_offset = offset + 1 + run_length_size
165
          run_offset = attribute[offset_run_offset, run_offset_size]
166
          if run_offset[-1].ord & 128 == 128
167
            run_offset += "\xFF" * (8 - run_offset_size)
168
          else
169
            run_offset += "\x00" * (8 - run_offset_size)
170
          end
171
          run_offset = run_offset.unpack('q<')[0]
172
          #offset relative to previous offset
173
          run_offset += old_offset
174

175
          size_wanted = [run_length * @bytes_per_cluster, size_max - attribut.length].min
176
          if cluster_num + (size_max / @bytes_per_cluster) >= run_list_num && (cluster_num < run_length + run_list_num)
177
            run_list_offset_in_cluster = run_offset + [cluster_num - run_list_num, 0].max
178
            run_list_offset = (run_list_offset_in_cluster) * @bytes_per_cluster
179
            run_list_offset = run_list_offset.to_i
180
            @file_handler.seek(run_list_offset)
181

182
            data = ''
183
            while data.length < size_wanted
184
              # Use a 4Mb block size to avoid target memory consumption
185
              data << @file_handler.read([size_wanted - data.length, 2**22].min)
186
            end
187
            attribut << data
188
          end
189
          offset += run_offset_size + run_length_size + 1
190
          run_list_num += run_length
191
          old_offset = run_offset
192
        end
193
        attribut = attribut[0, real_size]
194
        attribut
195
      end
196

197
      #
198
      # return the attribute list from the MFT record
199
      # deal with resident and non resident attributes (but not $DATA due to performance issue)
200
      # if lazy = True, this function only gather essential non resident attributes
201
      # (INDEX_ALLOCATION). Non resident attributes can still be gathered later with
202
      # cluster_from_attribute_non_resident function.
203
      #
204
      def mft_record_attribute(mft_record, lazy=true)
205
        attribute_list_offset = mft_record[20, 2].unpack('C')[0]
206
        curs = attribute_list_offset
207
        attribute_identifier = mft_record[curs, 4].unpack('V')[0]
208
        res = {}
209
        while attribute_identifier != 0xFFFFFFFF
210
          # attribute_size=mft_record[curs + 4, 4].unpack('V')[0]
211
          # should be on 4 bytes but doesnt work
212
          attribute_size = mft_record[curs + 4, 2].unpack('v')[0]
213
          # resident
214
          if mft_record[curs + 8] == "\x00"
215
            content_size = mft_record[curs + 16, 4].unpack('V')[0]
216
            content_offset = mft_record[curs + 20, 2].unpack('v')[0]
217
            res[attribute_identifier] = mft_record[curs + content_offset, content_size]
218
          else
219
            # non resident
220
            if attribute_identifier == INDEX_ALLOCATION_ID or
221
              (!lazy and attribute_identifier != DATA_ATTRIBUTE_ID)
222
              res[attribute_identifier] = cluster_from_attribute_non_resident(mft_record[curs, attribute_size])
223
            else
224
              res[attribute_identifier] = mft_record[curs, attribute_size]
225
            end
226
          end
227
          if attribute_identifier == DATA_ATTRIBUTE_ID
228
            res[attribute_identifier] = {
229
              'data' => res[attribute_identifier],
230
              'resident' => mft_record[curs + 8] == "\x00" }
231
          end
232
          curs += attribute_size
233
          attribute_identifier = mft_record[curs, 4].unpack('V')[0]
234
        end
235
        res
236
      end
237

238
      #
239
      # return the file path in the NTFS partition
240
      #
241
      def file(path)
242
        repertory = mft_record_from_mft_num(5)
243
        index_entry = {}
244
        path.split('\\').each do |r|
245
          attributes = mft_record_attribute(repertory)
246
          index = index_list_from_attributes(attributes)
247
          unless index.key?(r)
248
            fail ArgumentError, 'File path does not exist', caller
249
          end
250
          index_entry = index[r]
251
          repertory = mft_record_from_mft_num(index_entry['mft_offset'])
252
        end
253
        file_content_from_mft_num(index_entry['mft_offset'], index_entry['file_size'])
254
      end
255
    end
256
  end
257
end
258

259