Main Page | Class Hierarchy | Data Structures | File List | Data Fields | Globals | Related Pages

AdvFileAttr.cpp File Reference

Detailed Description

This class provides eMule with functions to handle conversion of files to Sparse files along with file-system level compression/decompression support.

For terms and conditions, read Terms.dox. Distribution is allowed as GPL as long as these comments remain included verbatim.

	
You are free to use and modify this file to the extent allowed by the GPL at the exclusive condition that you send me
a notification of your intent to use or modify this file. Send such notifications to digitalmastrmind_at_hotmail_dot_com
or visit http://pages.infinit.net/moonligh/eMule for other details. I would appreciate it if you would submit
modifications to me for implementation instead of implementing changes yourself. Comments and suggestions are welcome as well.

All disclosed source code is considered GPL'd unless otherwise noted.
All files created specifically for building my Doxygen-based projects web site (.dox) remain my exclusive property.
All my own files contain a reference to this text and should be treated as if this was replicated in each of them.
If you change my files, add comments below my header telling me who you are, what you changed and why so I know who
to give credits to when I update my files.

Modification of this file (Terms.txt) is not allowed.
Distribution is allowed as GPL as long this file is untouched and accompanies my own source files that refer to it.

- Moonlight.

Author:
Moonlight (digitalmastrmind@at@hotmail.dot.com), September-October 2003
Version:
0.70-test1
Note:
This is the real full-Monty Sparse&Compression support job... quite a different story from my MakeSparse() quick hack!

This is the completely rewritten version I started writing about in late October. This version only uses a single thread and an item queue, the whole thing being in WorkQueue.h and WorkQueue.cpp (CWorkQueue and CWorkQueueItem)

Last changed: 2003-11-05 - Finishing up the single-thread + queue re-implementation, you may want to take a look at CWorkQueue and CWorkQueueItem.

Contents:

AdvFileAttr.cpp 

#include "StdAfx.h"
#include "emule.h"
#include "AdvFileAttr.h"

#define CTL_CODE( DeviceType, Function, Method, Access ) (                 \
    ((DeviceType) << 16) | ((Access) << 14) | ((Function) << 2) | (Method) \
)
#define METHOD_BUFFERED                 0
#define FILE_ANY_ACCESS                 0
#define FILE_SPECIAL_ACCESS    (FILE_ANY_ACCESS)
#define FILE_DEVICE_FILE_SYSTEM         0x00000009
#define FSCTL_SET_ZERO_DATA             CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 50, METHOD_BUFFERED, FILE_WRITE_DATA) // FILE_ZERO_DATA_INFORMATION,
#define FSCTL_SET_SPARSE                CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 49, METHOD_BUFFERED, FILE_SPECIAL_ACCESS)
#define FSCTL_SET_COMPRESSION           CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 16, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA)


#define AFA_ID_SHUTDOWN     0
#define AFA_ID_CANCEL       1
#define AFA_ID_START        2


typedef struct _FILE_ZERO_DATA_INFORMATION {

    LARGE_INTEGER FileOffset;
    LARGE_INTEGER BeyondFinalZero;

} FILE_ZERO_DATA_INFORMATION, *PFILE_ZERO_DATA_INFORMATION;

CWorkQueue  CAdvFileAttr::m_WorkQueue(&ShutdownEvent);

CAdvFileAttr::CAdvFileAttr(CFile *file) : m_LastOp(TOps_Nothing) {
    SetFilePtr(file);
}

CAdvFileAttr::~CAdvFileAttr(void) {
    Cancel();
    while (isBusy()) Sleep(0);
}

void    CAdvFileAttr::Cancel(void) {
    CCriticalSection tempSection;
    tempSection.Lock();
    if (m_LastOp != TOps_Nothing && !m_WorkQueue.DelItem(this)) m_LastOp = TOps_Cancel;
}

void    CAdvFileAttr::SetFilePtr(CFile *file) {
    m_pCFile = file;
    if (!m_pCFile || (m_pCFile->m_hFile == m_pCFile->hFileNull) ) return;

    LoadAttributes();
    CString tempPath = m_pCFile->GetFilePath();
    if (!GetVolumeInformation(tempPath.Left(3), NULL, 0, NULL, NULL, (DWORD*)&m_TempFSCapabilities, NULL, 0) ) {
        CString temp;
        temp.Format("Unable to get file system capabilities for \"%s\"", tempPath);
        theApp.AddDebugLogLine(false,  temp);
        m_TempFSCapabilities = 0;
    } 
}

bool    CAdvFileAttr::LoadAttributes() {
    // invalid handle or file
    if ( !m_pCFile || (m_pCFile->m_hFile == m_pCFile->hFileNull) ) {
        m_FileAttributes = 0;
        return false;
    } else {
        m_FileAttributes = ::GetFileAttributes(m_pCFile->GetFilePath());
        return  true;
    }
}

void CAdvFileAttr::WorkCompress(void) {
    DWORD dummy;
    USHORT compression = isCompressed() ? COMPRESSION_FORMAT_NONE : COMPRESSION_FORMAT_DEFAULT;
    if(!DeviceIoControl(m_pCFile->m_hFile, FSCTL_SET_COMPRESSION, &compression, sizeof(compression), NULL, 0, &dummy, NULL)) {
        CString temp;
        temp.Format("Unable to %s \"%s\".", compression == COMPRESSION_FORMAT_NONE ? "Deompress":"Compress", m_pCFile->GetFilePath());
        if (!ShutdownEvent.isShuttingDown()) theApp.AddDebugLogLine(false,  temp);
    }
    LoadAttributes();
}

bool CAdvFileAttr::MakeCompressed(bool makeCompressed) {
    // Check attributes.
    if (!isFSCompressed() || (isCompressed() == makeCompressed)) return false;

    // Check all the basic essentials...
    if (!m_pCFile || (m_pCFile->m_hFile == CFile::hFileNull)) return false;

    m_LastOp = TOps_Compress;
    return m_WorkQueue.AddItem(this);
}

bool CAdvFileAttr::WorkSparse(void) {
    // Check for null pointers and invalid handles one last time before proceeding any further.
    if (ShutdownEvent.isShuttingDown() || !m_pCFile || (m_pCFile->m_hFile == CFile::hFileNull)) return false;

    CString temp;
    DWORD   dummy;
    CFile   WorkFile(m_pCFile->GetFilePath(), CFile::modeReadWrite | CFile::shareDenyNone);

    temp.Format("Sparse Scan: \"%s\", %.2fMB, this may take a while.", WorkFile.GetFilePath(), WorkFile.GetLength() / 1048576.0F);
    theApp.AddDebugLogLine(false,  temp);

    // Use a relatively small block size to avoid stalling other threads for too long or being stalled myself if other threads start locking
    // ranges at some later point in time. 256KB should be pretty quick to scan.
    enum {  zBufLen = 1<<18 };
    uint8   *zBuffer = new uint8[zBufLen];

    uint32      zRead, zScan;   // read bytes and scan position counters.
    bool        zMode;          // keep track of the current 'mode', false = scan for 0, true = scan for non-zero.
    ULONGLONG   zIndex = 0;     // Locking range start index, start at 0.
    ULONGLONG   zLIndex = 0;    // Locking range start index, start at 0.
    uint8       retryCount;     // Lock retry counter.
    bool        abort = false;  // true if there is an error with DeviceIoControl to signal quitting.
    FILE_ZERO_DATA_INFORMATION  zStartEnd;
    WorkFile.SeekToBegin();
        
    // Start scanning for zeroes, file indexes start at zero as well.
    zMode = false;
    zStartEnd.FileOffset.QuadPart = zStartEnd.BeyondFinalZero.QuadPart = 0;
    
    do {
        // Retry for as many as 20 times.
        retryCount  = 20;
        zRead = 0;
        while (--retryCount && !zRead) try {
            if (ShutdownEvent.isShuttingDown() || isCancel()) {
                abort = true;
                break;
            }
            zStartEnd.FileOffset.QuadPart = zStartEnd.BeyondFinalZero.QuadPart = zIndex = WorkFile.GetPosition();
            if (zLIndex > zIndex) theApp.AddDebugLogLine(false,  "Sparse Scan: Gone backwards... oops.");
            zLIndex = zIndex;
            WorkFile.LockRange(zIndex, zBufLen);
            zRead = WorkFile.Read(zBuffer, zBufLen);

        } catch (CFileException *e) {
            if( e->m_cause == e->lockViolation ) {
                e->Delete();
                Sleep(0);
            } else if ( e->m_cause == e->endOfFile ) {
                e->Delete();
                break;
            } else {
                abort = true;
                retryCount = 0;
                temp.Format("- ERROR %i scanning the file - Aborted.", e->m_cause);
                theApp.AddDebugLogLine(false,  temp);
                e->Delete();
                break;
            }
        }

        zScan = 0;
        if (retryCount && zRead && !abort) {    // Managed to lock the range? Then scan it.
            while ((zScan < zRead) && !abort) {
                if (!zMode) {
                    // Avoid doing LONGLONG++ by simply playing with offsets.
                    zStartEnd.FileOffset.QuadPart -= zScan;
                    while((zScan < zRead) && (zBuffer[zScan])) zScan++; // Scan until zero.
                    zStartEnd.FileOffset.QuadPart += zScan;

                    if (zScan < zRead) {    // If a zero was found, switch in non-zero scan mode.
                        zStartEnd.BeyondFinalZero.QuadPart = zStartEnd.FileOffset.QuadPart;
                        zMode = true;
                    }
                }

                if (zMode) {
                    zStartEnd.BeyondFinalZero.QuadPart -= zScan;
                    while((zScan < zRead) && (!zBuffer[zScan])) zScan++;    // Scan until nonzero
                    zStartEnd.BeyondFinalZero.QuadPart += zScan;

                    // It might not be worth bothering with less than 1KB... but I'll bother with 64 bytes just to be sure.
                    if (zStartEnd.BeyondFinalZero.QuadPart - zStartEnd.FileOffset.QuadPart >= 64)
                        if (!DeviceIoControl(WorkFile.m_hFile,FSCTL_SET_ZERO_DATA, &zStartEnd, sizeof(zStartEnd), NULL, 0,  &dummy, NULL)) {
                            temp.Format("- ERROR %i scanning the file - Aborted.", GetLastError());
                            if (!ShutdownEvent.isShuttingDown()) theApp.AddDebugLogLine(false,  temp);
                            abort = true;
                        }
                    zMode = false;
                    zStartEnd.FileOffset.QuadPart = zStartEnd.BeyondFinalZero.QuadPart;
                }
            }
        }
        // Release the file lock.
        try {
            WorkFile.UnlockRange(zIndex, zBufLen);
        } catch (CFileException *e) {
            if (e->m_cause == e->generic) { // Generic error - happens when the file gets closed during shutdown.
                e->Delete();
                break;
            } else throw e;
        }
        zMode = false;
    } while (zRead == zBufLen && !abort && !ShutdownEvent.isShuttingDown());

    CString extraStatus = "Completed";
    if (ShutdownEvent.isShuttingDown()) extraStatus = "Shutting down";
    else if (isCancel()) extraStatus = "Incomplete (cancelled)";
    else if (abort) extraStatus = "Incomplete (error)";

    // Get the new 'compressed' size to compare with the original.
    zStartEnd.FileOffset.LowPart = GetCompressedFileSize(WorkFile.GetFilePath(), (DWORD*)&zStartEnd.FileOffset.HighPart);
    temp.Format("- Raw size : %I64i, On-Disk size : %I64i (%.4f:1) - %s", WorkFile.GetLength(), zStartEnd.FileOffset.QuadPart, 1.0f * zStartEnd.FileOffset.QuadPart / WorkFile.GetLength(), extraStatus);
    if (!ShutdownEvent.isShuttingDown()) theApp.AddDebugLogLine(false,  temp);
    delete[] zBuffer;
    return true;
}

bool CAdvFileAttr::MakeSparse(bool forceScan) {
    CString partfile = m_pCFile->GetFilePath();
    CString temp;
    DWORD   dummy;

    // Check attributes.
    if  ((isSparse() && !forceScan) || !isFSSparse() || isBusy()) return false;

    // Check all the basic essentials...
    if (!m_pCFile || (m_pCFile->m_hFile == CFile::hFileNull)) return false;
    
    if  ((!isSparse() && DeviceIoControl(m_pCFile->m_hFile,FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &dummy, NULL)) ||
        (isSparse() && forceScan)) {
        LoadAttributes();
        m_LastOp = TOps_Sparse;
        return m_WorkQueue.AddItem(this);
    } else {
        temp.Format("- Failed to make the file sparse, error %i", GetLastError());
        theApp.AddDebugLogLine(false,  temp);
        return false;
    }
}

void CAdvFileAttr::ThreadWork(void) {
    if      (ShutdownEvent.isShuttingDown())    return;
    else if (m_LastOp == TOps_Nothing)          return;
    else if (m_LastOp == TOps_Compress)         WorkCompress();
    else if (m_LastOp == TOps_Sparse)           WorkSparse();
    QueueDelCB();
}

#include "StdAfx.h"
#include "emule.h"
#include "AdvFileAttr.h"

Data Structures

struct  _FILE_ZERO_DATA_INFORMATION
 These lines were copy-pasted from WinIOCtl.h due to it generating undeclared identifiers.


These lines were copy-pasted from WinIOCtl.h due to it generating undeclared identifiers.

#define CTL_CODE(DeviceType, Function, Method, Access)
#define METHOD_BUFFERED   0
#define FILE_ANY_ACCESS   0
#define FILE_SPECIAL_ACCESS   (FILE_ANY_ACCESS)
#define FILE_DEVICE_FILE_SYSTEM   0x00000009
#define FSCTL_SET_ZERO_DATA   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 50, METHOD_BUFFERED, FILE_WRITE_DATA)
#define FSCTL_SET_SPARSE   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 49, METHOD_BUFFERED, FILE_SPECIAL_ACCESS)
#define FSCTL_SET_COMPRESSION   CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 16, METHOD_BUFFERED, FILE_READ_DATA | FILE_WRITE_DATA)

Lock object IDs for CMultiLock

#define AFA_ID_SHUTDOWN   0
#define AFA_ID_CANCEL   1
#define AFA_ID_START   2

Typedefs

typedef _FILE_ZERO_DATA_INFORMATION FILE_ZERO_DATA_INFORMATION
 These lines were copy-pasted from WinIOCtl.h due to it generating undeclared identifiers.

typedef _FILE_ZERO_DATA_INFORMATION * PFILE_ZERO_DATA_INFORMATION
 These lines were copy-pasted from WinIOCtl.h due to it generating undeclared identifiers.

Define Documentation

#define CTL_CODE (  DeviceType,
Function,
Method,
Access   ) 
 

Value:

(                 \
    ((DeviceType) << 16) | ((Access) << 14) | ((Function) << 2) | (Method) \
)

Generated on Wed Nov 5 05:48:08 2003 for Moonlight's eMule Tweaks Documentation by doxygen 1.3.4