[NTFS] - Commit early results of a small restructuring effort:

-Add a new member to the NTFS_ATTR_CONTEXT struct, a LARGE_MCB. This allows an attribute context to describe the cluster mapping of a non-resident file while allowing that mapping to change dynamically, without the context itself needing to be resized. This fixes problems which sometimes arose from resizing files.
-Remove hacky code from NtfsWriteFile() for dealing with "stale" contexts. This fixes that issue.
-Update SetDataAttributeLength(), PrepareAttributeContext(), ReleaseAttributeContext(), FreeClusters(), and AddRun() for the new member.
-Update ReadAttribute() and WriteAttribute() to work with the changed structure. A very-soon-to-come commit will overhaul these functions so they'll operate directly on the LARGE_MCB, instead of converting to and from a packed list of data runs. (Sparse files are broken until then.)
-Rename "RunBufferOffset" to "RunBufferSize" in several places where appropriate.
-Fix, improve, and add some comments.

svn path=/branches/GSoC_2016/NTFS/; revision=74523
This commit is contained in:
Trevor Thompson 2017-05-12 22:16:20 +00:00 committed by Thomas Faber
parent 0409b3161e
commit 52b9f46776
4 changed files with 128 additions and 115 deletions

View file

@ -261,20 +261,33 @@ AddRun(PNTFS_VCB Vcb,
ULONG RunLength)
{
NTSTATUS Status;
PUCHAR DataRun = (PUCHAR)&AttrContext->Record + AttrContext->Record.NonResident.MappingPairsOffset;
int DataRunMaxLength;
PNTFS_ATTR_RECORD DestinationAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + AttrOffset);
LARGE_MCB DataRunsMCB;
ULONG NextAttributeOffset = AttrOffset + AttrContext->Record.Length;
ULONGLONG NextVBN = AttrContext->Record.NonResident.LowestVCN;
ULONGLONG NextVBN = 0;
// Allocate some memory for the RunBuffer
PUCHAR RunBuffer;
ULONG RunBufferOffset = 0;
ULONG RunBufferSize;
if (!AttrContext->Record.IsNonResident)
return STATUS_INVALID_PARAMETER;
if (AttrContext->Record.NonResident.AllocatedSize != 0)
NextVBN = AttrContext->Record.NonResident.HighestVCN + 1;
// Add newly-assigned clusters to mcb
_SEH2_TRY{
if (!FsRtlAddLargeMcbEntry(&AttrContext->DataRunsMCB,
NextVBN,
NextAssignedCluster,
RunLength))
{
ExRaiseStatus(STATUS_UNSUCCESSFUL);
}
} _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) {
_SEH2_YIELD(_SEH2_GetExceptionCode());
} _SEH2_END;
RunBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS);
if (!RunBuffer)
{
@ -282,89 +295,55 @@ AddRun(PNTFS_VCB Vcb,
return STATUS_INSUFFICIENT_RESOURCES;
}
// Convert the data runs to a map control block
Status = ConvertDataRunsToLargeMCB(DataRun, &DataRunsMCB, &NextVBN);
if (!NT_SUCCESS(Status))
{
DPRINT1("Unable to convert data runs to MCB (probably ran out of memory)!\n");
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
return Status;
}
// Add newly-assigned clusters to mcb
_SEH2_TRY{
if (!FsRtlAddLargeMcbEntry(&DataRunsMCB,
NextVBN,
NextAssignedCluster,
RunLength))
{
ExRaiseStatus(STATUS_UNSUCCESSFUL);
}
} _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) {
FsRtlUninitializeLargeMcb(&DataRunsMCB);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
_SEH2_YIELD(_SEH2_GetExceptionCode());
} _SEH2_END;
// Convert the map control block back to encoded data runs
ConvertLargeMCBToDataRuns(&DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferOffset);
ConvertLargeMCBToDataRuns(&AttrContext->DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferSize);
// Get the amount of free space between the start of the of the first data run and the attribute end
DataRunMaxLength = AttrContext->Record.Length - AttrContext->Record.NonResident.MappingPairsOffset;
// Do we need to extend the attribute (or convert to attribute list)?
if (DataRunMaxLength < RunBufferOffset)
if (DataRunMaxLength < RunBufferSize)
{
PNTFS_ATTR_RECORD NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset);
DataRunMaxLength += Vcb->NtfsInfo.BytesPerFileRecord - NextAttributeOffset - (sizeof(ULONG) * 2);
// Can we move the end of the attribute?
if (NextAttribute->Type != AttributeEnd || DataRunMaxLength < RunBufferOffset - 1)
if (NextAttribute->Type != AttributeEnd || DataRunMaxLength < RunBufferSize - 1)
{
DPRINT1("FIXME: Need to create attribute list! Max Data Run Length available: %d\n", DataRunMaxLength);
if (NextAttribute->Type != AttributeEnd)
DPRINT1("There's another attribute after this one with type %0xlx\n", NextAttribute->Type);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
FsRtlUninitializeLargeMcb(&DataRunsMCB);
return STATUS_NOT_IMPLEMENTED;
}
// calculate position of end markers
NextAttributeOffset = AttrOffset + AttrContext->Record.NonResident.MappingPairsOffset + RunBufferOffset;
NextAttributeOffset = AttrOffset + AttrContext->Record.NonResident.MappingPairsOffset + RunBufferSize;
NextAttributeOffset = ALIGN_UP_BY(NextAttributeOffset, 8);
// Write the end markers
NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset);
NextAttribute->Type = AttributeEnd;
NextAttribute->Length = FILE_RECORD_END;
// Update the length
DestinationAttribute->Length = NextAttributeOffset - AttrOffset;
AttrContext->Record.Length = DestinationAttribute->Length;
// We need to increase the FileRecord size
FileRecord->BytesInUse = NextAttributeOffset + (sizeof(ULONG) * 2);
// End the file record
NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset);
SetFileRecordEnd(FileRecord, NextAttribute, FILE_RECORD_END);
}
// NOTE: from this point on the original attribute record will contain invalid data in it's runbuffer
// TODO: Elegant fix? Could we free the old Record and allocate a new one without issue?
// Update HighestVCN
DestinationAttribute->NonResident.HighestVCN =
AttrContext->Record.NonResident.HighestVCN = max(NextVBN - 1 + RunLength,
AttrContext->Record.NonResident.HighestVCN = max(NextVBN - 1 + RunLength,
AttrContext->Record.NonResident.HighestVCN);
// Write data runs to destination attribute
RtlCopyMemory((PVOID)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset),
RunBuffer,
RunBufferOffset);
RunBufferSize);
// Update the file record
Status = UpdateFileRecord(Vcb, AttrContext->FileMFTIndex, FileRecord);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
FsRtlUninitializeLargeMcb(&DataRunsMCB);
NtfsDumpDataRuns((PUCHAR)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset), 0);
@ -567,7 +546,7 @@ ConvertLargeMCBToDataRuns(PLARGE_MCB DataRunsMCB,
DataRunLengthSize = GetPackedByteCount(Count, TRUE);
DPRINT("%d bytes needed.\n", DataRunLengthSize);
// ensure the next data run + end marker would be > Max buffer size
// ensure the next data run + end marker would be <= Max buffer size
if (RunBufferOffset + 2 + DataRunLengthSize + DataRunOffsetSize > MaxBufferSize)
{
Status = STATUS_BUFFER_TOO_SMALL;
@ -698,17 +677,12 @@ FreeClusters(PNTFS_VCB Vcb,
NTSTATUS Status = STATUS_SUCCESS;
ULONG ClustersLeftToFree = ClustersToFree;
// convert data runs to mcb
PUCHAR DataRun = (PUCHAR)&AttrContext->Record + AttrContext->Record.NonResident.MappingPairsOffset;
PNTFS_ATTR_RECORD DestinationAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + AttrOffset);
LARGE_MCB DataRunsMCB;
ULONG NextAttributeOffset = AttrOffset + AttrContext->Record.Length;
PNTFS_ATTR_RECORD NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)FileRecord + NextAttributeOffset);
ULONGLONG NextVBN = AttrContext->Record.NonResident.LowestVCN;
// Allocate some memory for the RunBuffer
PUCHAR RunBuffer;
ULONG RunBufferOffset = 0;
ULONG RunBufferSize = 0;
PFILE_RECORD_HEADER BitmapRecord;
PNTFS_ATTR_CONTEXT DataContext;
@ -722,30 +696,13 @@ FreeClusters(PNTFS_VCB Vcb,
return STATUS_INVALID_PARAMETER;
}
RunBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS);
if (!RunBuffer)
{
DPRINT1("ERROR: Couldn't allocate memory for data runs!\n");
return STATUS_INSUFFICIENT_RESOURCES;
}
// Convert the data runs to a map control block
Status = ConvertDataRunsToLargeMCB(DataRun, &DataRunsMCB, &NextVBN);
if (!NT_SUCCESS(Status))
{
DPRINT1("Unable to convert data runs to MCB (probably ran out of memory)!\n");
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
return Status;
}
// Read the $Bitmap file
BitmapRecord = ExAllocatePoolWithTag(NonPagedPool,
Vcb->NtfsInfo.BytesPerFileRecord,
TAG_NTFS);
if (BitmapRecord == NULL)
{
DPRINT1("Error: Unable to allocate memory for bitmap file record!\n");
FsRtlUninitializeLargeMcb(&DataRunsMCB);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
return STATUS_NO_MEMORY;
}
@ -753,9 +710,7 @@ FreeClusters(PNTFS_VCB Vcb,
if (!NT_SUCCESS(Status))
{
DPRINT1("Error: Unable to read file record for bitmap!\n");
FsRtlUninitializeLargeMcb(&DataRunsMCB);
ExFreePoolWithTag(BitmapRecord, TAG_NTFS);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
return 0;
}
@ -763,9 +718,7 @@ FreeClusters(PNTFS_VCB Vcb,
if (!NT_SUCCESS(Status))
{
DPRINT1("Error: Unable to find data attribute for bitmap file!\n");
FsRtlUninitializeLargeMcb(&DataRunsMCB);
ExFreePoolWithTag(BitmapRecord, TAG_NTFS);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
return 0;
}
@ -777,9 +730,7 @@ FreeClusters(PNTFS_VCB Vcb,
{
DPRINT1("Error: Unable to allocate memory for bitmap file data!\n");
ReleaseAttributeContext(DataContext);
FsRtlUninitializeLargeMcb(&DataRunsMCB);
ExFreePoolWithTag(BitmapRecord, TAG_NTFS);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
return 0;
}
@ -792,7 +743,7 @@ FreeClusters(PNTFS_VCB Vcb,
{
LONGLONG LargeVbn, LargeLbn;
if (!FsRtlLookupLastLargeMcbEntry(&DataRunsMCB, &LargeVbn, &LargeLbn))
if (!FsRtlLookupLastLargeMcbEntry(&AttrContext->DataRunsMCB, &LargeVbn, &LargeLbn))
{
Status = STATUS_INVALID_PARAMETER;
DPRINT1("DRIVER ERROR: FreeClusters called to free %lu clusters, which is %lu more clusters than are assigned to attribute!",
@ -806,7 +757,9 @@ FreeClusters(PNTFS_VCB Vcb,
// deallocate this cluster
RtlClearBits(&Bitmap, LargeLbn, 1);
}
FsRtlTruncateLargeMcb(&DataRunsMCB, AttrContext->Record.NonResident.HighestVCN);
FsRtlTruncateLargeMcb(&AttrContext->DataRunsMCB, AttrContext->Record.NonResident.HighestVCN);
// decrement HighestVCN, but don't let it go below 0
AttrContext->Record.NonResident.HighestVCN = min(AttrContext->Record.NonResident.HighestVCN, AttrContext->Record.NonResident.HighestVCN - 1);
ClustersLeftToFree--;
}
@ -816,19 +769,27 @@ FreeClusters(PNTFS_VCB Vcb,
if (!NT_SUCCESS(Status))
{
ReleaseAttributeContext(DataContext);
FsRtlUninitializeLargeMcb(&DataRunsMCB);
ExFreePoolWithTag(BitmapData, TAG_NTFS);
ExFreePoolWithTag(BitmapRecord, TAG_NTFS);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
return Status;
}
ReleaseAttributeContext(DataContext);
ExFreePoolWithTag(BitmapData, TAG_NTFS);
ExFreePoolWithTag(BitmapRecord, TAG_NTFS);
// Save updated data runs to file record
// Allocate some memory for a new RunBuffer
RunBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS);
if (!RunBuffer)
{
DPRINT1("ERROR: Couldn't allocate memory for data runs!\n");
return STATUS_INSUFFICIENT_RESOURCES;
}
// Convert the map control block back to encoded data runs
ConvertLargeMCBToDataRuns(&DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferOffset);
ConvertLargeMCBToDataRuns(&AttrContext->DataRunsMCB, RunBuffer, Vcb->NtfsInfo.BytesPerCluster, &RunBufferSize);
// Update HighestVCN
DestinationAttribute->NonResident.HighestVCN = AttrContext->Record.NonResident.HighestVCN;
@ -836,27 +797,23 @@ FreeClusters(PNTFS_VCB Vcb,
// Write data runs to destination attribute
RtlCopyMemory((PVOID)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset),
RunBuffer,
RunBufferOffset);
RunBufferSize);
// Is DestinationAttribute the last attribute in the file record?
if (NextAttribute->Type == AttributeEnd)
{
// update attribute length
AttrContext->Record.Length = ALIGN_UP_BY(AttrContext->Record.NonResident.MappingPairsOffset + RunBufferOffset, 8);
AttrContext->Record.Length = ALIGN_UP_BY(AttrContext->Record.NonResident.MappingPairsOffset + RunBufferSize, 8);
DestinationAttribute->Length = AttrContext->Record.Length;
// write end markers
NextAttribute = (PNTFS_ATTR_RECORD)((ULONG_PTR)DestinationAttribute + DestinationAttribute->Length);
NextAttribute->Type = AttributeEnd;
NextAttribute->Length = FILE_RECORD_END;
// update file record length
FileRecord->BytesInUse = AttrOffset + DestinationAttribute->Length + (sizeof(ULONG) * 2);
SetFileRecordEnd(FileRecord, NextAttribute, FILE_RECORD_END);
}
// Update the file record
Status = UpdateFileRecord(Vcb, AttrContext->FileMFTIndex, FileRecord);
FsRtlUninitializeLargeMcb(&DataRunsMCB);
ExFreePoolWithTag(RunBuffer, TAG_NTFS);
NtfsDumpDataRuns((PUCHAR)((ULONG_PTR)DestinationAttribute + DestinationAttribute->NonResident.MappingPairsOffset), 0);

View file

@ -50,8 +50,10 @@ PrepareAttributeContext(PNTFS_ATTR_RECORD AttrRecord)
{
LONGLONG DataRunOffset;
ULONGLONG DataRunLength;
ULONGLONG NextVBN = 0;
PUCHAR DataRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset;
Context->CacheRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset;
Context->CacheRun = DataRun;
Context->CacheRunOffset = 0;
Context->CacheRun = DecodeRun(Context->CacheRun, &DataRunOffset, &DataRunLength);
Context->CacheRunLength = DataRunLength;
@ -68,6 +70,14 @@ PrepareAttributeContext(PNTFS_ATTR_RECORD AttrRecord)
Context->CacheRunLastLCN = 0;
}
Context->CacheRunCurrentOffset = 0;
// Convert the data runs to a map control block
if (!NT_SUCCESS(ConvertDataRunsToLargeMCB(DataRun, &Context->DataRunsMCB, &NextVBN)))
{
DPRINT1("Unable to convert data runs to MCB!\n");
ExFreePoolWithTag(Context, TAG_NTFS);
return NULL;
}
}
return Context;
@ -77,6 +87,11 @@ PrepareAttributeContext(PNTFS_ATTR_RECORD AttrRecord)
VOID
ReleaseAttributeContext(PNTFS_ATTR_CONTEXT Context)
{
if (Context->Record.IsNonResident)
{
FsRtlUninitializeLargeMcb(&Context->DataRunsMCB);
}
ExFreePoolWithTag(Context, TAG_NTFS);
}
@ -246,10 +261,30 @@ SetAttributeDataLength(PFILE_OBJECT FileObject,
ULONG NextAssignedCluster;
ULONG AssignedClusters;
NTSTATUS Status = GetLastClusterInDataRun(Fcb->Vcb, &AttrContext->Record, (PULONGLONG)&LastClusterInDataRun.QuadPart);
if (ExistingClusters == 0)
{
LastClusterInDataRun.QuadPart = 0;
}
else
{
if (!FsRtlLookupLargeMcbEntry(&AttrContext->DataRunsMCB,
(LONGLONG)AttrContext->Record.NonResident.HighestVCN,
(PLONGLONG)&LastClusterInDataRun.QuadPart,
NULL,
NULL,
NULL,
NULL))
{
DPRINT1("Error looking up final large MCB entry!\n");
DPRINT1("GetLastClusterInDataRun returned: %I64u\n", LastClusterInDataRun.QuadPart);
DPRINT1("Highest VCN of record: %I64u\n", AttrContext->Record.NonResident.HighestVCN);
// Most likely, HighestVCN went above the largest mapping
DPRINT1("Highest VCN of record: %I64u\n", AttrContext->Record.NonResident.HighestVCN);
return STATUS_INVALID_PARAMETER;
}
}
DPRINT("LastClusterInDataRun: %I64u\n", LastClusterInDataRun.QuadPart);
DPRINT("Highest VCN of record: %I64u\n", AttrContext->Record.NonResident.HighestVCN);
while (ClustersNeeded > 0)
{
@ -405,6 +440,9 @@ ReadAttribute(PDEVICE_EXTENSION Vcb,
ULONG ReadLength;
ULONG AlreadyRead;
NTSTATUS Status;
//TEMPTEMP
PUCHAR TempBuffer;
if (!Context->Record.IsNonResident)
{
@ -438,10 +476,21 @@ ReadAttribute(PDEVICE_EXTENSION Vcb,
}
else
{
//TEMPTEMP
ULONG UsedBufferSize;
TempBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS);
LastLCN = 0;
DataRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset;
CurrentOffset = 0;
// This will be rewritten in the next iteration to just use the DataRuns MCB directly
ConvertLargeMCBToDataRuns(&Context->DataRunsMCB,
TempBuffer,
Vcb->NtfsInfo.BytesPerFileRecord,
&UsedBufferSize);
DataRun = TempBuffer;
while (1)
{
DataRun = DecodeRun(DataRun, &DataRunOffset, &DataRunLength);
@ -558,6 +607,10 @@ ReadAttribute(PDEVICE_EXTENSION Vcb,
} /* if Disk */
// TEMPTEMP
if (Context->Record.IsNonResident)
ExFreePoolWithTag(TempBuffer, TAG_NTFS);
Context->CacheRun = DataRun;
Context->CacheRunOffset = Offset + AlreadyRead;
Context->CacheRunStartLCN = DataRunStartLCN;
@ -622,6 +675,10 @@ WriteAttribute(PDEVICE_EXTENSION Vcb,
NTSTATUS Status;
PUCHAR SourceBuffer = Buffer;
LONGLONG StartingOffset;
//TEMPTEMP
PUCHAR TempBuffer;
DPRINT("WriteAttribute(%p, %p, %I64u, %p, %lu, %p)\n", Vcb, Context, Offset, Buffer, Length, RealLengthWritten);
@ -707,9 +764,19 @@ WriteAttribute(PDEVICE_EXTENSION Vcb,
}
else*/
{
ULONG UsedBufferSize;
LastLCN = 0;
DataRun = (PUCHAR)&Context->Record + Context->Record.NonResident.MappingPairsOffset;
CurrentOffset = 0;
CurrentOffset = 0;
// This will be rewritten in the next iteration to just use the DataRuns MCB directly
TempBuffer = ExAllocatePoolWithTag(NonPagedPool, Vcb->NtfsInfo.BytesPerFileRecord, TAG_NTFS);
ConvertLargeMCBToDataRuns(&Context->DataRunsMCB,
TempBuffer,
Vcb->NtfsInfo.BytesPerFileRecord,
&UsedBufferSize);
DataRun = TempBuffer;
while (1)
{
@ -864,6 +931,10 @@ WriteAttribute(PDEVICE_EXTENSION Vcb,
}
} // end while (Length > 0) [more data to write]
// TEMPTEMP
if(Context->Record.IsNonResident)
ExFreePoolWithTag(TempBuffer, TAG_NTFS);
Context->CacheRun = DataRun;
Context->CacheRunOffset = Offset + *RealLengthWritten;
Context->CacheRunStartLCN = DataRunStartLCN;

View file

@ -437,6 +437,7 @@ typedef struct _NTFS_ATTR_CONTEXT
ULONGLONG CacheRunLength;
LONGLONG CacheRunLastLCN;
ULONGLONG CacheRunCurrentOffset;
LARGE_MCB DataRunsMCB;
ULONGLONG FileMFTIndex;
NTFS_ATTR_RECORD Record;
} NTFS_ATTR_CONTEXT, *PNTFS_ATTR_CONTEXT;

View file

@ -432,22 +432,6 @@ NTSTATUS NtfsWriteFile(PDEVICE_EXTENSION DeviceExt,
return Status;
}
// at this point the record in DataContext may be stale, so we need to refresh it
ReleaseAttributeContext(DataContext);
Status = FindAttribute(DeviceExt,
FileRecord,
AttributeData,
Fcb->Stream,
wcslen(Fcb->Stream),
&DataContext,
&AttributeOffset);
if (!NT_SUCCESS(Status))
{
DPRINT1("DRIVER ERROR: Couldn't find $DATA attribute after setting size!\n");
return Status;
}
// now we need to update this file's size in every directory index entry that references it
// TODO: put this code in its own function and adapt it to work with every filename / hardlink
// stored in the file record.