缓冲文件(用于更快的磁盘访问)

前端之家收集整理的这篇文章主要介绍了缓冲文件(用于更快的磁盘访问)前端之家小编觉得挺不错的,现在分享给大家,也给大家做个参考。
我正在处理大文件和直接写入磁盘是慢的。因为文件是大的,我不能加载它在一个TMemoryStream。

TFileStream不缓冲,所以我想知道是否有一个自定义库,可以提供缓冲流或我应该只依赖于操作系统提供的缓冲。操作系统缓冲是否可靠?我的意思是如果缓存已满,可能会从缓存中清除旧文件(我的),以便为新文件腾出空间。

我的文件在GB范围内。它包含数百万条记录。不幸的是,记录不是固定大小。所以,我必须做数百万的读数(在4和500字节之间)。读(和写)是顺序的。我不上下跳入文件(我认为是理想的缓冲)。

最后,我必须把这样的文件写回磁盘(再次上百万的小写)。

对大卫赫弗南的赞美!
David提供了一个提供缓冲磁盘访问的大量代码
你必须拥有BufferedFileStream!它是金。不要忘记upvote。

谢谢大卫。

解决方法

Windows文件缓存非常有效,特别是如果你使用Vista或更高版本。 TFileStream是围绕Windows ReadFile()和WriteFile()API函数的松散包装,对于许多使用情况,唯一更快的是内存映射文件

然而,有一个常见的情况,TFileStream成为性能瓶颈。也就是说,如果每次调用流读取或写入函数读取或写入少量数据。例如,如果你一次读取一个整数数组一个项目,那么在调用ReadFile()时,一次读取4个字节会产生很大的开销。

再次,内存映射文件解决这个瓶颈的一个很好的方法,但另一个常用的方法是读取一个更大的缓冲区,许多千字节说,然后解决流从内存缓存中的未来读取,而不是进一步调用ReadFile()。这种方法只适用于顺序访问。

从更新的问题中描述的使用模式,我认为你可能会发现以下类将提高你的性能

  1. unit BufferedFileStream;
  2.  
  3. interface
  4.  
  5. uses
  6. SysUtils,Math,Classes,Windows;
  7.  
  8. type
  9. TBaseCachedFileStream = class(TStream)
  10. private
  11. function QueryInterface(const IID: TGUID; out Obj): HResult; stdcall;
  12. function _AddRef: Integer; stdcall;
  13. function _Release: Integer; stdcall;
  14. protected
  15. FHandle: THandle;
  16. FOwnsHandle: Boolean;
  17. FCache: PByte;
  18. FCacheSize: Integer;
  19. FPosition: Int64;//the current position in the file (relative to the beginning of the file)
  20. FCacheStart: Int64;//the postion in the file of the start of the cache (relative to the beginning of the file)
  21. FCacheEnd: Int64;//the postion in the file of the end of the cache (relative to the beginning of the file)
  22. FFileName: string;
  23. FLastError: DWORD;
  24. procedure HandleError(const Msg: string);
  25. procedure RaiseSystemError(const Msg: string; LastError: DWORD); overload;
  26. procedure RaiseSystemError(const Msg: string); overload;
  27. procedure RaiseSystemErrorFmt(const Msg: string; const Args: array of const);
  28. function CreateHandle(FlagsAndAttributes: DWORD): THandle; virtual; abstract;
  29. function GetFileSize: Int64; virtual;
  30. procedure SetSize(NewSize: Longint); override;
  31. procedure SetSize(const NewSize: Int64); override;
  32. function FileRead(var Buffer; Count: Longword): Integer;
  33. function FileWrite(const Buffer; Count: Longword): Integer;
  34. function FileSeek(const Offset: Int64; Origin: TSeekOrigin): Int64;
  35. public
  36. constructor Create(const FileName: string); overload;
  37. constructor Create(const FileName: string; CacheSize: Integer); overload;
  38. constructor Create(const FileName: string; CacheSize: Integer; Handle: THandle); overload; virtual;
  39. destructor Destroy; override;
  40. property CacheSize: Integer read FCacheSize;
  41. function Read(var Buffer; Count: Longint): Longint; override;
  42. function Write(const Buffer; Count: Longint): Longint; override;
  43. function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override;
  44. end;
  45. TBaseCachedFileStreamClass = class of TBaseCachedFileStream;
  46.  
  47. IDisableStreamReadCache = interface
  48. ['{0B6D0004-88D1-42D5-BC0F-447911C0FC21}']
  49. procedure DisableStreamReadCache;
  50. procedure EnableStreamReadCache;
  51. end;
  52.  
  53. TReadOnlyCachedFileStream = class(TBaseCachedFileStream,IDisableStreamReadCache)
  54. (* This class works by filling the cache each time a call to Read is made and
  55. FPosition is outside the existing cache. By filling the cache we mean
  56. reading from the file into the temporary cache. Calls to Read when
  57. FPosition is in the existing cache are then dealt with by filling the
  58. buffer with bytes from the cache.
  59. *)
  60. private
  61. FUseAlignedCache: Boolean;
  62. FViewStart: Int64;
  63. FViewLength: Int64;
  64. FDisableStreamReadCacheRefCount: Integer;
  65. procedure DisableStreamReadCache;
  66. procedure EnableStreamReadCache;
  67. procedure FlushCache;
  68. protected
  69. function CreateHandle(FlagsAndAttributes: DWORD): THandle; override;
  70. function GetFileSize: Int64; override;
  71. public
  72. constructor Create(const FileName: string; CacheSize: Integer; Handle: THandle); overload; override;
  73. property UseAlignedCache: Boolean read FUseAlignedCache write FUseAlignedCache;
  74. function Read(var Buffer; Count: Longint): Longint; override;
  75. procedure SetViewWindow(const ViewStart,ViewLength: Int64);
  76. end;
  77.  
  78. TWriteCachedFileStream = class(TBaseCachedFileStream,IDisableStreamReadCache)
  79. (* This class works by caching calls to Write. By this we mean temporarily
  80. storing the bytes to be written in the cache. As each call to Write is
  81. processed the cache grows. The cache is written to file when:
  82. 1. A call to Write is made when the cache is full.
  83. 2. A call to Write is made and FPosition is outside the cache (this
  84. must be as a result of a call to Seek).
  85. 3. The class is destroyed.
  86.  
  87. Note that data can be read from these streams but the reading is not
  88. cached and in fact a read operation will flush the cache before
  89. attempting to read the data.
  90. *)
  91. private
  92. FFileSize: Int64;
  93. FReadStream: TReadOnlyCachedFileStream;
  94. FReadStreamCacheSize: Integer;
  95. FReadStreamUseAlignedCache: Boolean;
  96. procedure DisableStreamReadCache;
  97. procedure EnableStreamReadCache;
  98. procedure CreateReadStream;
  99. procedure FlushCache;
  100. protected
  101. function CreateHandle(FlagsAndAttributes: DWORD): THandle; override;
  102. function GetFileSize: Int64; override;
  103. public
  104. constructor Create(const FileName: string; CacheSize,ReadStreamCacheSize: Integer; ReadStreamUseAlignedCache: Boolean); overload;
  105. destructor Destroy; override;
  106. function Read(var Buffer; Count: Longint): Longint; override;
  107. function Write(const Buffer; Count: Longint): Longint; override;
  108. end;
  109.  
  110. implementation
  111.  
  112. function GetFileSizeEx(hFile: THandle; var FileSize: Int64): BOOL; stdcall; external kernel32;
  113. function SetFilePointerEx(hFile: THandle; DistanceToMove: Int64; lpNewFilePointer: PInt64; dwMoveMethod: DWORD): BOOL; stdcall; external kernel32;
  114.  
  115. { TBaseCachedFileStream }
  116.  
  117. constructor TBaseCachedFileStream.Create(const FileName: string);
  118. begin
  119. Create(FileName,0);
  120. end;
  121.  
  122. constructor TBaseCachedFileStream.Create(const FileName: string; CacheSize: Integer);
  123. begin
  124. Create(FileName,CacheSize,0);
  125. end;
  126.  
  127. constructor TBaseCachedFileStream.Create(const FileName: string; CacheSize: Integer; Handle: THandle);
  128. const
  129. DefaultCacheSize = 16*1024;
  130. //16kb - this was chosen empirically - don't make it too large otherwise the progress report is 'jerky'
  131. begin
  132. inherited Create;
  133. FFileName := FileName;
  134. FOwnsHandle := Handle=0;
  135. if FOwnsHandle then begin
  136. FHandle := CreateHandle(FILE_ATTRIBUTE_NORMAL);
  137. end else begin
  138. FHandle := Handle;
  139. end;
  140. FCacheSize := CacheSize;
  141. if FCacheSize<=0 then begin
  142. FCacheSize := DefaultCacheSize;
  143. end;
  144. GetMem(FCache,FCacheSize);
  145. end;
  146.  
  147. destructor TBaseCachedFileStream.Destroy;
  148. begin
  149. FreeMem(FCache);
  150. if FOwnsHandle and (FHandle<>0) then begin
  151. CloseHandle(FHandle);
  152. end;
  153. inherited;
  154. end;
  155.  
  156. function TBaseCachedFileStream.QueryInterface(const IID: TGUID; out Obj): HResult;
  157. begin
  158. if GetInterface(IID,Obj) then begin
  159. Result := S_OK;
  160. end else begin
  161. Result := E_NOINTERFACE;
  162. end;
  163. end;
  164.  
  165. function TBaseCachedFileStream._AddRef: Integer;
  166. begin
  167. Result := -1;
  168. end;
  169.  
  170. function TBaseCachedFileStream._Release: Integer;
  171. begin
  172. Result := -1;
  173. end;
  174.  
  175. procedure TBaseCachedFileStream.HandleError(const Msg: string);
  176. begin
  177. if FLastError<>0 then begin
  178. RaiseSystemError(Msg,FLastError);
  179. end;
  180. end;
  181.  
  182. procedure TBaseCachedFileStream.RaiseSystemError(const Msg: string; LastError: DWORD);
  183. begin
  184. raise EStreamError.Create(Trim(Msg+' ')+SysErrorMessage(LastError));
  185. end;
  186.  
  187. procedure TBaseCachedFileStream.RaiseSystemError(const Msg: string);
  188. begin
  189. RaiseSystemError(Msg,GetLastError);
  190. end;
  191.  
  192. procedure TBaseCachedFileStream.RaiseSystemErrorFmt(const Msg: string; const Args: array of const);
  193. begin
  194. RaiseSystemError(Format(Msg,Args));
  195. end;
  196.  
  197. function TBaseCachedFileStream.GetFileSize: Int64;
  198. begin
  199. if not GetFileSizeEx(FHandle,Result) then begin
  200. RaiseSystemErrorFmt('GetFileSizeEx Failed for %s.',[FFileName]);
  201. end;
  202. end;
  203.  
  204. procedure TBaseCachedFileStream.SetSize(NewSize: Longint);
  205. begin
  206. SetSize(Int64(NewSize));
  207. end;
  208.  
  209. procedure TBaseCachedFileStream.SetSize(const NewSize: Int64);
  210. begin
  211. Seek(NewSize,soBeginning);
  212. if not Windows.SetEndOfFile(FHandle) then begin
  213. RaiseSystemErrorFmt('SetEndOfFile for %s.',[FFileName]);
  214. end;
  215. end;
  216.  
  217. function TBaseCachedFileStream.FileRead(var Buffer; Count: Longword): Integer;
  218. begin
  219. if Windows.ReadFile(FHandle,Buffer,Count,LongWord(Result),nil) then begin
  220. FLastError := 0;
  221. end else begin
  222. FLastError := GetLastError;
  223. Result := -1;
  224. end;
  225. end;
  226.  
  227. function TBaseCachedFileStream.FileWrite(const Buffer; Count: Longword): Integer;
  228. begin
  229. if Windows.WriteFile(FHandle,nil) then begin
  230. FLastError := 0;
  231. end else begin
  232. FLastError := GetLastError;
  233. Result := -1;
  234. end;
  235. end;
  236.  
  237. function TBaseCachedFileStream.FileSeek(const Offset: Int64; Origin: TSeekOrigin): Int64;
  238. begin
  239. if not SetFilePointerEx(FHandle,Offset,@Result,ord(Origin)) then begin
  240. RaiseSystemErrorFmt('SetFilePointerEx Failed for %s.',[FFileName]);
  241. end;
  242. end;
  243.  
  244. function TBaseCachedFileStream.Read(var Buffer; Count: Integer): Longint;
  245. begin
  246. raise EAssertionFailed.Create('Cannot read from this stream');
  247. end;
  248.  
  249. function TBaseCachedFileStream.Write(const Buffer; Count: Integer): Longint;
  250. begin
  251. raise EAssertionFailed.Create('Cannot write to this stream');
  252. end;
  253.  
  254. function TBaseCachedFileStream.Seek(const Offset: Int64; Origin: TSeekOrigin): Int64;
  255. //Set FPosition to the value specified - if this has implications for the
  256. //cache then overriden Write and Read methods must deal with those.
  257. begin
  258. case Origin of
  259. soBeginning:
  260. FPosition := Offset;
  261. soEnd:
  262. FPosition := GetFileSize+Offset;
  263. soCurrent:
  264. inc(FPosition,Offset);
  265. end;
  266. Result := FPosition;
  267. end;
  268.  
  269. { TReadOnlyCachedFileStream }
  270.  
  271. constructor TReadOnlyCachedFileStream.Create(const FileName: string; CacheSize: Integer; Handle: THandle);
  272. begin
  273. inherited;
  274. SetViewWindow(0,inherited GetFileSize);
  275. end;
  276.  
  277. function TReadOnlyCachedFileStream.CreateHandle(FlagsAndAttributes: DWORD): THandle;
  278. begin
  279. Result := Windows.CreateFile(
  280. PChar(FFileName),GENERIC_READ,FILE_SHARE_READ,nil,OPEN_EXISTING,FlagsAndAttributes,0
  281. );
  282. if Result=INVALID_HANDLE_VALUE then begin
  283. RaiseSystemErrorFmt('Cannot open %s.',[FFileName]);
  284. end;
  285. end;
  286.  
  287. procedure TReadOnlyCachedFileStream.DisableStreamReadCache;
  288. begin
  289. inc(FDisableStreamReadCacheRefCount);
  290. end;
  291.  
  292. procedure TReadOnlyCachedFileStream.EnableStreamReadCache;
  293. begin
  294. dec(FDisableStreamReadCacheRefCount);
  295. end;
  296.  
  297. procedure TReadOnlyCachedFileStream.FlushCache;
  298. begin
  299. FCacheStart := 0;
  300. FCacheEnd := 0;
  301. end;
  302.  
  303. function TReadOnlyCachedFileStream.GetFileSize: Int64;
  304. begin
  305. Result := FViewLength;
  306. end;
  307.  
  308. procedure TReadOnlyCachedFileStream.SetViewWindow(const ViewStart,ViewLength: Int64);
  309. begin
  310. if ViewStart<0 then begin
  311. raise EAssertionFailed.Create('Invalid view window');
  312. end;
  313. if (ViewStart+ViewLength)>inherited GetFileSize then begin
  314. raise EAssertionFailed.Create('Invalid view window');
  315. end;
  316. FViewStart := ViewStart;
  317. FViewLength := ViewLength;
  318. FPosition := 0;
  319. FCacheStart := 0;
  320. FCacheEnd := 0;
  321. end;
  322.  
  323. function TReadOnlyCachedFileStream.Read(var Buffer; Count: Longint): Longint;
  324. var
  325. NumOfBytesToCopy,NumOfBytesLeft,NumOfBytesRead: Longint;
  326. CachePtr,BufferPtr: PByte;
  327. begin
  328. if FDisableStreamReadCacheRefCount>0 then begin
  329. FileSeek(FPosition+FViewStart,soBeginning);
  330. Result := FileRead(Buffer,Count);
  331. if Result=-1 then begin
  332. Result := 0;//contract is to return number of bytes that were read
  333. end;
  334. inc(FPosition,Result);
  335. end else begin
  336. Result := 0;
  337. NumOfBytesLeft := Count;
  338. BufferPtr := @Buffer;
  339. while NumOfBytesLeft>0 do begin
  340. if (FPosition<FCacheStart) or (FPosition>=FCacheEnd) then begin
  341. //the current position is not available in the cache so we need to re-fill the cache
  342. FCacheStart := FPosition;
  343. if UseAlignedCache then begin
  344. FCacheStart := FCacheStart - (FCacheStart mod CacheSize);
  345. end;
  346. FileSeek(FCacheStart+FViewStart,soBeginning);
  347. NumOfBytesRead := FileRead(FCache^,CacheSize);
  348. if NumOfBytesRead=-1 then begin
  349. exit;
  350. end;
  351. Assert(NumOfBytesRead>=0);
  352. FCacheEnd := FCacheStart+NumOfBytesRead;
  353. if NumOfBytesRead=0 then begin
  354. FLastError := ERROR_HANDLE_EOF;//must be at the end of the file
  355. break;
  356. end;
  357. end;
  358.  
  359. //read from cache to Buffer
  360. NumOfBytesToCopy := Min(FCacheEnd-FPosition,NumOfBytesLeft);
  361. CachePtr := FCache;
  362. inc(CachePtr,FPosition-FCacheStart);
  363. Move(CachePtr^,BufferPtr^,NumOfBytesToCopy);
  364. inc(Result,NumOfBytesToCopy);
  365. inc(FPosition,NumOfBytesToCopy);
  366. inc(BufferPtr,NumOfBytesToCopy);
  367. dec(NumOfBytesLeft,NumOfBytesToCopy);
  368. end;
  369. end;
  370. end;
  371.  
  372. { TWriteCachedFileStream }
  373.  
  374. constructor TWriteCachedFileStream.Create(const FileName: string; CacheSize,ReadStreamCacheSize: Integer; ReadStreamUseAlignedCache: Boolean);
  375. begin
  376. inherited Create(FileName,CacheSize);
  377. FReadStreamCacheSize := ReadStreamCacheSize;
  378. FReadStreamUseAlignedCache := ReadStreamUseAlignedCache;
  379. end;
  380.  
  381. destructor TWriteCachedFileStream.Destroy;
  382. begin
  383. FlushCache;//make sure that the final calls to Write get recorded in the file
  384. FreeAndNil(FReadStream);
  385. inherited;
  386. end;
  387.  
  388. function TWriteCachedFileStream.CreateHandle(FlagsAndAttributes: DWORD): THandle;
  389. begin
  390. Result := Windows.CreateFile(
  391. PChar(FFileName),GENERIC_READ or GENERIC_WRITE,CREATE_ALWAYS,0
  392. );
  393. if Result=INVALID_HANDLE_VALUE then begin
  394. RaiseSystemErrorFmt('Cannot create %s.',[FFileName]);
  395. end;
  396. end;
  397.  
  398. procedure TWriteCachedFileStream.DisableStreamReadCache;
  399. begin
  400. CreateReadStream;
  401. FReadStream.DisableStreamReadCache;
  402. end;
  403.  
  404. procedure TWriteCachedFileStream.EnableStreamReadCache;
  405. begin
  406. Assert(Assigned(FReadStream));
  407. FReadStream.EnableStreamReadCache;
  408. end;
  409.  
  410. function TWriteCachedFileStream.GetFileSize: Int64;
  411. begin
  412. Result := FFileSize;
  413. end;
  414.  
  415. procedure TWriteCachedFileStream.CreateReadStream;
  416. begin
  417. if not Assigned(FReadStream) then begin
  418. FReadStream := TReadOnlyCachedFileStream.Create(FFileName,FReadStreamCacheSize,FHandle);
  419. FReadStream.UseAlignedCache := FReadStreamUseAlignedCache;
  420. end;
  421. end;
  422.  
  423. procedure TWriteCachedFileStream.FlushCache;
  424. var
  425. NumOfBytesToWrite: Longint;
  426. begin
  427. if Assigned(FCache) then begin
  428. NumOfBytesToWrite := FCacheEnd-FCacheStart;
  429. if NumOfBytesToWrite>0 then begin
  430. FileSeek(FCacheStart,soBeginning);
  431. if FileWrite(FCache^,NumOfBytesToWrite)<>NumOfBytesToWrite then begin
  432. RaiseSystemErrorFmt('FileWrite Failed for %s.',[FFileName]);
  433. end;
  434. if Assigned(FReadStream) then begin
  435. FReadStream.FlushCache;
  436. end;
  437. end;
  438. FCacheStart := FPosition;
  439. FCacheEnd := FPosition;
  440. end;
  441. end;
  442.  
  443. function TWriteCachedFileStream.Read(var Buffer; Count: Integer): Longint;
  444. begin
  445. FlushCache;
  446. CreateReadStream;
  447. Assert(FReadStream.FViewStart=0);
  448. if FReadStream.FViewLength<>FFileSize then begin
  449. FReadStream.SetViewWindow(0,FFileSize);
  450. end;
  451. FReadStream.Position := FPosition;
  452. Result := FReadStream.Read(Buffer,Count);
  453. inc(FPosition,Result);
  454. end;
  455.  
  456. function TWriteCachedFileStream.Write(const Buffer; Count: Longint): Longint;
  457. var
  458. NumOfBytesToCopy,NumOfBytesLeft: Longint;
  459. CachePtr,BufferPtr: PByte;
  460. begin
  461. Result := 0;
  462. NumOfBytesLeft := Count;
  463. BufferPtr := @Buffer;
  464. while NumOfBytesLeft>0 do begin
  465. if ((FPosition<FCacheStart) or (FPosition>FCacheEnd))//the current position is outside the cache
  466. or (FPosition-FCacheStart=FCacheSize)//the cache is full
  467. then begin
  468. FlushCache;
  469. Assert(FCacheStart=FPosition);
  470. end;
  471.  
  472. //write from Buffer to the cache
  473. NumOfBytesToCopy := Min(FCacheSize-(FPosition-FCacheStart),NumOfBytesLeft);
  474. CachePtr := FCache;
  475. inc(CachePtr,FPosition-FCacheStart);
  476. Move(BufferPtr^,CachePtr^,NumOfBytesToCopy);
  477. inc(Result,NumOfBytesToCopy);
  478. inc(FPosition,NumOfBytesToCopy);
  479. FCacheEnd := Max(FCacheEnd,FPosition);
  480. inc(BufferPtr,NumOfBytesToCopy);
  481. dec(NumOfBytesLeft,NumOfBytesToCopy);
  482. end;
  483. FFileSize := Max(FFileSize,FPosition);
  484. end;
  485.  
  486. end.

猜你在找的Delphi相关文章