2018-02-07 17:10:59 +00:00
|
|
|
/*
|
|
|
|
* COPYRIGHT: See COPYING in the top level directory
|
|
|
|
* PROJECT: ReactOS kernel
|
|
|
|
* FILE: ntoskrnl/cc/lazywrite.c
|
|
|
|
* PURPOSE: Cache manager
|
|
|
|
*
|
|
|
|
* PROGRAMMERS: Pierre Schweitzer (pierre@reactos.org)
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* INCLUDES *****************************************************************/
|
|
|
|
|
|
|
|
#include <ntoskrnl.h>
|
|
|
|
#define NDEBUG
|
|
|
|
#include <debug.h>
|
|
|
|
|
|
|
|
/* Counters:
|
|
|
|
* - Amount of pages flushed by lazy writer
|
|
|
|
* - Number of times lazy writer ran
|
|
|
|
*/
|
|
|
|
ULONG CcLazyWritePages = 0;
|
|
|
|
ULONG CcLazyWriteIos = 0;
|
|
|
|
|
|
|
|
/* Internal vars (MS):
|
|
|
|
* - Lazy writer status structure
|
|
|
|
* - Lookaside list where to allocate work items
|
2018-02-09 09:06:17 +00:00
|
|
|
* - Queue for high priority work items (read ahead)
|
2018-02-07 17:10:59 +00:00
|
|
|
* - Queue for regular work items
|
|
|
|
* - Available worker threads
|
2018-02-07 17:56:13 +00:00
|
|
|
* - Queue for stuff to be queued after lazy writer is done
|
2018-02-07 17:10:59 +00:00
|
|
|
* - Marker for throttling queues
|
|
|
|
* - Number of ongoing workers
|
|
|
|
* - Three seconds delay for lazy writer
|
|
|
|
* - One second delay for lazy writer
|
|
|
|
* - Zero delay for lazy writer
|
|
|
|
* - Number of worker threads
|
|
|
|
*/
|
|
|
|
LAZY_WRITER LazyWriter;
|
|
|
|
NPAGED_LOOKASIDE_LIST CcTwilightLookasideList;
|
2018-02-09 09:06:17 +00:00
|
|
|
LIST_ENTRY CcExpressWorkQueue;
|
2018-02-07 17:10:59 +00:00
|
|
|
LIST_ENTRY CcRegularWorkQueue;
|
|
|
|
LIST_ENTRY CcIdleWorkerThreadList;
|
2018-02-07 17:56:13 +00:00
|
|
|
LIST_ENTRY CcPostTickWorkQueue;
|
2018-02-07 17:10:59 +00:00
|
|
|
BOOLEAN CcQueueThrottle = FALSE;
|
|
|
|
ULONG CcNumberActiveWorkerThreads = 0;
|
|
|
|
LARGE_INTEGER CcFirstDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*3000*1000*10);
|
|
|
|
LARGE_INTEGER CcIdleDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*1000*1000*10);
|
|
|
|
LARGE_INTEGER CcNoDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)0);
|
|
|
|
ULONG CcNumberWorkerThreads;
|
|
|
|
|
|
|
|
/* FUNCTIONS *****************************************************************/
|
|
|
|
|
|
|
|
VOID
|
|
|
|
CcPostWorkQueue(
|
|
|
|
IN PWORK_QUEUE_ENTRY WorkItem,
|
|
|
|
IN PLIST_ENTRY WorkQueue)
|
|
|
|
{
|
|
|
|
KIRQL OldIrql;
|
|
|
|
PWORK_QUEUE_ITEM ThreadToSpawn;
|
|
|
|
|
|
|
|
/* First of all, insert the item in the queue */
|
|
|
|
OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock);
|
|
|
|
InsertTailList(WorkQueue, &WorkItem->WorkQueueLinks);
|
|
|
|
|
|
|
|
/* Now, define whether we have to spawn a new work thread
|
|
|
|
* We will spawn a new one if:
|
|
|
|
* - There's no throttle in action
|
|
|
|
* - There's still at least one idle thread
|
|
|
|
*/
|
|
|
|
ThreadToSpawn = NULL;
|
|
|
|
if (!CcQueueThrottle && !IsListEmpty(&CcIdleWorkerThreadList))
|
|
|
|
{
|
|
|
|
PLIST_ENTRY ListEntry;
|
|
|
|
|
|
|
|
/* Get the idle thread */
|
|
|
|
ListEntry = RemoveHeadList(&CcIdleWorkerThreadList);
|
|
|
|
ThreadToSpawn = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ITEM, List);
|
|
|
|
|
|
|
|
/* We're going to have one more! */
|
|
|
|
CcNumberActiveWorkerThreads += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
|
|
|
|
|
|
|
|
/* If we have a thread to spawn, do it! */
|
|
|
|
if (ThreadToSpawn != NULL)
|
|
|
|
{
|
|
|
|
/* We NULLify it to be consistent with initialization */
|
|
|
|
ThreadToSpawn->List.Flink = NULL;
|
|
|
|
ExQueueWorkItem(ThreadToSpawn, CriticalWorkQueue);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
CcScanDpc(
|
|
|
|
IN PKDPC Dpc,
|
|
|
|
IN PVOID DeferredContext,
|
|
|
|
IN PVOID SystemArgument1,
|
|
|
|
IN PVOID SystemArgument2)
|
|
|
|
{
|
|
|
|
PWORK_QUEUE_ENTRY WorkItem;
|
|
|
|
|
|
|
|
/* Allocate a work item */
|
|
|
|
WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
|
|
|
|
if (WorkItem == NULL)
|
|
|
|
{
|
|
|
|
LazyWriter.ScanActive = FALSE;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* And post it, it will be for lazy write */
|
2018-12-23 13:43:17 +00:00
|
|
|
WorkItem->Function = LazyScan;
|
2018-02-07 17:10:59 +00:00
|
|
|
CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
|
|
|
|
}
|
|
|
|
|
2018-12-23 13:43:17 +00:00
|
|
|
VOID
|
|
|
|
CcWriteBehind(VOID)
|
|
|
|
{
|
|
|
|
ULONG Target, Count;
|
|
|
|
|
|
|
|
Target = CcTotalDirtyPages / 8;
|
|
|
|
if (Target != 0)
|
|
|
|
{
|
|
|
|
/* Flush! */
|
|
|
|
DPRINT("Lazy writer starting (%d)\n", Target);
|
|
|
|
CcRosFlushDirtyPages(Target, &Count, FALSE, TRUE);
|
|
|
|
|
|
|
|
/* And update stats */
|
|
|
|
CcLazyWritePages += Count;
|
|
|
|
++CcLazyWriteIos;
|
|
|
|
DPRINT("Lazy writer done (%d)\n", Count);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-07 17:10:59 +00:00
|
|
|
VOID
|
|
|
|
CcLazyWriteScan(VOID)
|
|
|
|
{
|
|
|
|
ULONG Target;
|
2018-02-07 17:56:13 +00:00
|
|
|
KIRQL OldIrql;
|
2018-02-07 17:10:59 +00:00
|
|
|
PLIST_ENTRY ListEntry;
|
2018-02-07 17:56:13 +00:00
|
|
|
LIST_ENTRY ToPost;
|
|
|
|
PWORK_QUEUE_ENTRY WorkItem;
|
2018-02-07 17:10:59 +00:00
|
|
|
|
2018-02-07 17:56:13 +00:00
|
|
|
/* Do we have entries to queue after we're done? */
|
|
|
|
InitializeListHead(&ToPost);
|
|
|
|
OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
|
|
|
|
if (LazyWriter.OtherWork)
|
|
|
|
{
|
|
|
|
while (!IsListEmpty(&CcPostTickWorkQueue))
|
|
|
|
{
|
|
|
|
ListEntry = RemoveHeadList(&CcPostTickWorkQueue);
|
|
|
|
WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks);
|
|
|
|
InsertTailList(&ToPost, &WorkItem->WorkQueueLinks);
|
|
|
|
}
|
|
|
|
LazyWriter.OtherWork = FALSE;
|
|
|
|
}
|
|
|
|
KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
|
2018-02-07 17:10:59 +00:00
|
|
|
|
|
|
|
/* Our target is one-eighth of the dirty pages */
|
|
|
|
Target = CcTotalDirtyPages / 8;
|
|
|
|
if (Target != 0)
|
|
|
|
{
|
2018-12-23 13:43:17 +00:00
|
|
|
/* There is stuff to flush, schedule a write-behind operation */
|
2018-02-07 17:10:59 +00:00
|
|
|
|
2018-12-23 13:43:17 +00:00
|
|
|
/* Allocate a work item */
|
|
|
|
WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
|
|
|
|
if (WorkItem != NULL)
|
|
|
|
{
|
|
|
|
WorkItem->Function = WriteBehind;
|
|
|
|
CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
|
|
|
|
}
|
2018-02-07 17:10:59 +00:00
|
|
|
}
|
|
|
|
|
2018-02-07 20:35:22 +00:00
|
|
|
/* Post items that were due for end of run */
|
2018-02-07 17:56:13 +00:00
|
|
|
while (!IsListEmpty(&ToPost))
|
|
|
|
{
|
|
|
|
ListEntry = RemoveHeadList(&ToPost);
|
|
|
|
WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks);
|
|
|
|
CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
|
|
|
|
}
|
|
|
|
|
2018-05-02 21:30:10 +00:00
|
|
|
/* If we have deferred writes, try them now! */
|
|
|
|
if (!IsListEmpty(&CcDeferredWrites))
|
|
|
|
{
|
|
|
|
CcPostDeferredWrites();
|
|
|
|
/* Reschedule immediately a lazy writer run
|
|
|
|
* Keep us active to have short idle delay
|
|
|
|
*/
|
|
|
|
CcScheduleLazyWriteScan(FALSE);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
/* We're no longer active */
|
|
|
|
OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
|
|
|
|
LazyWriter.ScanActive = FALSE;
|
|
|
|
KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
|
|
|
|
}
|
2018-02-07 17:10:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
VOID CcScheduleLazyWriteScan(
|
|
|
|
IN BOOLEAN NoDelay)
|
|
|
|
{
|
|
|
|
/* If no delay, immediately start lazy writer,
|
|
|
|
* no matter it was already started
|
|
|
|
*/
|
|
|
|
if (NoDelay)
|
|
|
|
{
|
|
|
|
LazyWriter.ScanActive = TRUE;
|
|
|
|
KeSetTimer(&LazyWriter.ScanTimer, CcNoDelay, &LazyWriter.ScanDpc);
|
|
|
|
}
|
|
|
|
/* Otherwise, if it's not running, just wait three seconds to start it */
|
|
|
|
else if (!LazyWriter.ScanActive)
|
|
|
|
{
|
|
|
|
LazyWriter.ScanActive = TRUE;
|
|
|
|
KeSetTimer(&LazyWriter.ScanTimer, CcFirstDelay, &LazyWriter.ScanDpc);
|
|
|
|
}
|
|
|
|
/* Finally, already running, so queue for the next second */
|
|
|
|
else
|
|
|
|
{
|
|
|
|
KeSetTimer(&LazyWriter.ScanTimer, CcIdleDelay, &LazyWriter.ScanDpc);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
CcWorkerThread(
|
|
|
|
IN PVOID Parameter)
|
|
|
|
{
|
|
|
|
KIRQL OldIrql;
|
2018-12-23 13:43:17 +00:00
|
|
|
BOOLEAN DropThrottle, WritePerformed;
|
2018-02-07 17:10:59 +00:00
|
|
|
PWORK_QUEUE_ITEM Item;
|
2018-02-10 22:49:33 +00:00
|
|
|
#if DBG
|
|
|
|
PIRP TopLevel;
|
|
|
|
#endif
|
2018-02-07 17:10:59 +00:00
|
|
|
|
|
|
|
/* Get back our thread item */
|
|
|
|
Item = Parameter;
|
|
|
|
/* And by default, don't touch throttle */
|
|
|
|
DropThrottle = FALSE;
|
2018-12-23 13:43:17 +00:00
|
|
|
/* No write performed */
|
|
|
|
WritePerformed = FALSE;
|
2018-02-07 17:10:59 +00:00
|
|
|
|
2018-02-10 22:49:33 +00:00
|
|
|
#if DBG
|
|
|
|
/* Top level IRP should be clean when started
|
|
|
|
* Save it to catch buggy drivers (or bugs!)
|
|
|
|
*/
|
|
|
|
TopLevel = IoGetTopLevelIrp();
|
|
|
|
if (TopLevel != NULL)
|
|
|
|
{
|
|
|
|
DPRINT1("(%p) TopLevel IRP for this thread: %p\n", PsGetCurrentThread(), TopLevel);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2018-02-07 17:10:59 +00:00
|
|
|
/* Loop till we have jobs */
|
|
|
|
while (TRUE)
|
|
|
|
{
|
|
|
|
PWORK_QUEUE_ENTRY WorkItem;
|
|
|
|
|
|
|
|
/* Lock queues */
|
|
|
|
OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock);
|
|
|
|
|
|
|
|
/* If we have to touch throttle, reset it now! */
|
|
|
|
if (DropThrottle)
|
|
|
|
{
|
|
|
|
CcQueueThrottle = FALSE;
|
|
|
|
DropThrottle = FALSE;
|
|
|
|
}
|
|
|
|
|
2018-02-09 09:06:17 +00:00
|
|
|
/* Check first if we have read ahead to do */
|
|
|
|
if (IsListEmpty(&CcExpressWorkQueue))
|
2018-02-07 17:10:59 +00:00
|
|
|
{
|
2018-02-09 09:06:17 +00:00
|
|
|
/* If not, check regular queue */
|
|
|
|
if (IsListEmpty(&CcRegularWorkQueue))
|
|
|
|
{
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
WorkItem = CONTAINING_RECORD(CcRegularWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
WorkItem = CONTAINING_RECORD(CcExpressWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
|
2018-02-07 17:10:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Get our work item, if someone is waiting for us to finish
|
|
|
|
* and we're not the only thread in queue
|
|
|
|
* then, quit running to let the others do
|
|
|
|
* and throttle so that noone starts till current activity is over
|
|
|
|
*/
|
|
|
|
if (WorkItem->Function == SetDone && CcNumberActiveWorkerThreads > 1)
|
|
|
|
{
|
|
|
|
CcQueueThrottle = TRUE;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Otherwise, remove current entry */
|
|
|
|
RemoveEntryList(&WorkItem->WorkQueueLinks);
|
|
|
|
KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
|
|
|
|
|
|
|
|
/* And handle it */
|
|
|
|
switch (WorkItem->Function)
|
|
|
|
{
|
2018-02-09 09:06:17 +00:00
|
|
|
case ReadAhead:
|
|
|
|
CcPerformReadAhead(WorkItem->Parameters.Read.FileObject);
|
|
|
|
break;
|
|
|
|
|
2018-12-23 13:43:17 +00:00
|
|
|
case WriteBehind:
|
2018-12-23 13:59:42 +00:00
|
|
|
PsGetCurrentThread()->MemoryMaker = 1;
|
2018-12-23 13:43:17 +00:00
|
|
|
CcWriteBehind();
|
2018-12-23 13:59:42 +00:00
|
|
|
PsGetCurrentThread()->MemoryMaker = 0;
|
2018-12-23 13:43:17 +00:00
|
|
|
WritePerformed = TRUE;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case LazyScan:
|
2018-02-07 17:10:59 +00:00
|
|
|
CcLazyWriteScan();
|
|
|
|
break;
|
|
|
|
|
|
|
|
case SetDone:
|
|
|
|
KeSetEvent(WorkItem->Parameters.Event.Event, IO_NO_INCREMENT, FALSE);
|
|
|
|
DropThrottle = TRUE;
|
|
|
|
break;
|
2018-02-09 09:06:17 +00:00
|
|
|
|
|
|
|
default:
|
|
|
|
DPRINT1("Ignored item: %p (%d)\n", WorkItem, WorkItem->Function);
|
|
|
|
break;
|
2018-02-07 17:10:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* And release the item */
|
|
|
|
ExFreeToNPagedLookasideList(&CcTwilightLookasideList, WorkItem);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Our thread is available again */
|
|
|
|
InsertTailList(&CcIdleWorkerThreadList, &Item->List);
|
|
|
|
/* One less worker */
|
|
|
|
--CcNumberActiveWorkerThreads;
|
|
|
|
KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
|
2018-02-10 22:49:33 +00:00
|
|
|
|
2018-12-23 13:43:17 +00:00
|
|
|
/* If there are pending write openations and we have at least 20 dirty pages */
|
|
|
|
if (!IsListEmpty(&CcDeferredWrites) && CcTotalDirtyPages >= 20)
|
|
|
|
{
|
|
|
|
/* And if we performed a write operation previously, then
|
|
|
|
* stress the system a bit and reschedule a scan to find
|
|
|
|
* stuff to write
|
|
|
|
*/
|
|
|
|
if (WritePerformed)
|
|
|
|
{
|
|
|
|
CcLazyWriteScan();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-10 22:49:33 +00:00
|
|
|
#if DBG
|
|
|
|
/* Top level shouldn't have changed */
|
|
|
|
if (TopLevel != IoGetTopLevelIrp())
|
|
|
|
{
|
|
|
|
DPRINT1("(%p) Mismatching TopLevel: %p, %p\n", PsGetCurrentThread(), TopLevel, IoGetTopLevelIrp());
|
|
|
|
}
|
|
|
|
#endif
|
2018-02-07 17:10:59 +00:00
|
|
|
}
|
2018-02-07 17:56:13 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* @implemented
|
|
|
|
*/
|
|
|
|
NTSTATUS
|
|
|
|
NTAPI
|
|
|
|
CcWaitForCurrentLazyWriterActivity (
|
|
|
|
VOID)
|
|
|
|
{
|
|
|
|
KIRQL OldIrql;
|
|
|
|
KEVENT WaitEvent;
|
|
|
|
PWORK_QUEUE_ENTRY WorkItem;
|
|
|
|
|
|
|
|
/* Allocate a work item */
|
|
|
|
WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
|
|
|
|
if (WorkItem == NULL)
|
|
|
|
{
|
|
|
|
return STATUS_INSUFFICIENT_RESOURCES;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We want lazy writer to set our event */
|
|
|
|
WorkItem->Function = SetDone;
|
|
|
|
KeInitializeEvent(&WaitEvent, NotificationEvent, FALSE);
|
|
|
|
WorkItem->Parameters.Event.Event = &WaitEvent;
|
|
|
|
|
|
|
|
/* Use the post tick queue */
|
|
|
|
OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
|
|
|
|
InsertTailList(&CcPostTickWorkQueue, &WorkItem->WorkQueueLinks);
|
|
|
|
|
|
|
|
/* Inform the lazy writer it will have to handle the post tick queue */
|
|
|
|
LazyWriter.OtherWork = TRUE;
|
|
|
|
/* And if it's not running, queue a lazy writer run
|
|
|
|
* And start it NOW, we want the response now
|
|
|
|
*/
|
|
|
|
if (!LazyWriter.ScanActive)
|
|
|
|
{
|
|
|
|
CcScheduleLazyWriteScan(TRUE);
|
|
|
|
}
|
|
|
|
|
|
|
|
KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
|
|
|
|
|
|
|
|
/* And now, wait until lazy writer replies */
|
|
|
|
return KeWaitForSingleObject(&WaitEvent, Executive, KernelMode, FALSE, NULL);
|
|
|
|
}
|