00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
#include "ki.h"
00031
00032
00033
00034
00035
00036
VOID
00037
KiChangeColorPageTarget (
00038 IN PULONG SignalDone,
00039 IN PVOID NewColor,
00040 IN PVOID OldColor,
00041 IN PVOID PageFrame
00042 );
00043
00044
VOID
00045
KiSweepDcacheTarget (
00046 IN PULONG SignalDone,
00047 IN PVOID Parameter1,
00048 IN PVOID Parameter2,
00049 IN PVOID Parameter3
00050 );
00051
00052
VOID
00053
KiSweepIcacheTarget (
00054 IN PULONG SignalDone,
00055 IN PVOID Parameter1,
00056 IN PVOID Parameter2,
00057 IN PVOID Parameter3
00058 );
00059
00060
VOID
00061
KiSweepIcacheRangeTarget (
00062 IN PULONG SignalDone,
00063 IN PVOID BaseAddress,
00064 IN PVOID Length,
00065 IN PVOID Parameter3
00066 );
00067
00068
VOID
00069
KiFlushIoBuffersTarget (
00070 IN PULONG SignalDone,
00071 IN PVOID Mdl,
00072 IN PVOID ReadOperation,
00073 IN PVOID DmaOperation
00074 );
00075
00076
VOID
00077 KeChangeColorPage (
00078 IN PVOID NewColor,
00079 IN PVOID OldColor,
00080 IN ULONG PageFrame
00081 )
00082
00083
00084
00085
00086
00087
00088
00089
00090
00091
00092
00093
00094
00095
00096
00097
00098
00099
00100
00101
00102
00103
00104
00105 {
00106 KIRQL OldIrql;
00107 KAFFINITY TargetProcessors;
00108
00109
ASSERT(KeGetCurrentIrql() <=
SYNCH_LEVEL);
00110
00111
00112
00113
00114
00115
#if !defined(NT_UP)
00116
00117 OldIrql = KeRaiseIrqlToSynchLevel();
00118
00119
00120
00121
00122
00123
00124 TargetProcessors =
KeActiveProcessors & PCR->NotMember;
00125
if (TargetProcessors != 0) {
00126
KiIpiSendPacket(TargetProcessors,
00127
KiChangeColorPageTarget,
00128 (PVOID)NewColor,
00129 (PVOID)OldColor,
00130 (PVOID)PageFrame);
00131 }
00132
00133
#endif
00134
00135
#ifdef COLORED_PAGES
00136
00137
00138
00139
00140 HalChangeColorPage(NewColor, OldColor, PageFrame);
00141
00142
#endif
00143
00144
00145
00146
00147
00148
00149
#if !defined(NT_UP)
00150
00151
if (TargetProcessors != 0) {
00152
KiIpiStallOnPacketTargets();
00153 }
00154
00155
00156
00157
00158
00159
KeLowerIrql(OldIrql);
00160
00161
#endif
00162
00163
return;
00164 }
00165
00166
VOID
00167
KiChangeColorPageTarget (
00168 IN PULONG SignalDone,
00169 IN PVOID NewColor,
00170 IN PVOID OldColor,
00171 IN PVOID PageFrame
00172 )
00173
00174
00175
00176
00177
00178
00179
00180
00181
00182
00183
00184
00185
00186
00187
00188
00189
00190
00191
00192
00193
00194
00195
00196
00197
00198
00199 {
00200
00201
00202
00203
00204
00205
00206
#if !defined(NT_UP)
00207
00208
#ifdef COLORED_PAGES
00209
HalChangeColorPage(NewColor, OldColor, (ULONG)PageFrame);
00210
#endif
00211
00212
KiIpiSignalPacketDone(SignalDone);
00213
00214
#endif
00215
00216
return;
00217 }
00218
00219
VOID
00220 KeSweepDcache (
00221 IN BOOLEAN AllProcessors
00222 )
00223
00224
00225
00226
00227
00228
00229
00230
00231
00232
00233
00234
00235
00236
00237
00238
00239
00240
00241
00242
00243
00244
00245
00246
00247 {
00248
00249 KIRQL OldIrql;
00250 KAFFINITY TargetProcessors;
00251
00252
ASSERT(KeGetCurrentIrql() <=
SYNCH_LEVEL);
00253
00254
00255
00256
00257
00258
#if !defined(NT_UP)
00259
00260 OldIrql = KeRaiseIrqlToSynchLevel();
00261
00262
00263
00264
00265
00266
00267 TargetProcessors =
KeActiveProcessors & PCR->NotMember;
00268
if (TargetProcessors != 0) {
00269
KiIpiSendPacket(TargetProcessors,
00270
KiSweepDcacheTarget,
00271
NULL,
00272
NULL,
00273
NULL);
00274 }
00275
00276
#endif
00277
00278
00279
00280
00281
00282 HalSweepDcache();
00283
00284
00285
00286
00287
00288
00289
#if !defined(NT_UP)
00290
00291
if (TargetProcessors != 0) {
00292
KiIpiStallOnPacketTargets();
00293 }
00294
00295
00296
00297
00298
00299
KeLowerIrql(OldIrql);
00300
00301
#endif
00302
00303
return;
00304 }
00305
00306
VOID
00307
KiSweepDcacheTarget (
00308 IN PULONG SignalDone,
00309 IN PVOID Parameter1,
00310 IN PVOID Parameter2,
00311 IN PVOID Parameter3
00312 )
00313
00314
00315
00316
00317
00318
00319
00320
00321
00322
00323
00324
00325
00326
00327
00328
00329
00330
00331
00332
00333
00334
00335
00336
00337
00338 {
00339
00340
00341
00342
00343
00344
00345
#if !defined(NT_UP)
00346
00347 HalSweepDcache();
00348
KiIpiSignalPacketDone(SignalDone);
00349
00350
#endif
00351
00352
return;
00353 }
00354
00355
VOID
00356 KeSweepIcache (
00357 IN BOOLEAN AllProcessors
00358 )
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369
00370
00371
00372
00373
00374
00375
00376
00377
00378
00379
00380
00381
00382
00383 {
00384
00385 KIRQL OldIrql;
00386 KAFFINITY TargetProcessors;
00387
00388
ASSERT(KeGetCurrentIrql() <=
SYNCH_LEVEL);
00389
00390
00391
00392
00393
00394
#if !defined(NT_UP)
00395
00396 OldIrql = KeRaiseIrqlToSynchLevel();
00397
00398
00399
00400
00401
00402
00403 TargetProcessors =
KeActiveProcessors & PCR->NotMember;
00404
if (TargetProcessors != 0) {
00405
KiIpiSendPacket(TargetProcessors,
00406
KiSweepIcacheTarget,
00407
NULL,
00408
NULL,
00409
NULL);
00410 }
00411
00412
#endif
00413
00414
00415
00416
00417
00418
00419
00420
if ( ( (KeGetPvr() >> 16 ) & 0xffff ) > 1 ) {
00421 HalSweepDcache();
00422 }
00423
00424 HalSweepIcache();
00425
00426
00427
00428
00429
00430
00431
#if !defined(NT_UP)
00432
00433
if (TargetProcessors != 0) {
00434
KiIpiStallOnPacketTargets();
00435 }
00436
00437
00438
00439
00440
00441
KeLowerIrql(OldIrql);
00442
00443
#endif
00444
00445
return;
00446 }
00447
00448
VOID
00449
KiSweepIcacheTarget (
00450 IN PULONG SignalDone,
00451 IN PVOID Parameter1,
00452 IN PVOID Parameter2,
00453 IN PVOID Parameter3
00454 )
00455
00456
00457
00458
00459
00460
00461
00462
00463
00464
00465
00466
00467
00468
00469
00470
00471
00472
00473
00474
00475
00476 {
00477
00478
00479
00480
00481
00482
00483
00484
00485
00486
if ( ( (KeGetPvr() >> 16 ) & 0xffff ) > 1 ) {
00487 HalSweepDcache();
00488 }
00489
00490
00491
#if !defined(NT_UP)
00492
00493 HalSweepIcache();
00494
KiIpiSignalPacketDone(SignalDone);
00495
00496
#endif
00497
00498
return;
00499 }
00500
00501
VOID
00502 KeSweepIcacheRange (
00503 IN BOOLEAN AllProcessors,
00504 IN PVOID BaseAddress,
00505 IN ULONG Length
00506 )
00507
00508
00509
00510
00511
00512
00513
00514
00515
00516
00517
00518
00519
00520
00521
00522
00523
00524
00525
00526
00527
00528
00529
00530
00531
00532
00533
00534 {
00535
00536 ULONG
Offset;
00537 KIRQL OldIrql;
00538 KAFFINITY TargetProcessors;
00539 ULONG ProcessorType;
00540 ULONG DcacheAlignment;
00541 ULONG IcacheAlignment;
00542
00543
ASSERT(KeGetCurrentIrql() <=
SYNCH_LEVEL);
00544
00545
00546
00547
00548
00549
00550
00551
00552
00553
if (Length > PCR->FirstLevelIcacheSize) {
00554
KeSweepIcache(AllProcessors);
00555
return;
00556 }
00557
00558 ProcessorType = KeGetPvr() >> 16;
00559
00560
if (ProcessorType != 1) {
00561
00562
00563
00564
00565
00566 DcacheAlignment = PCR->DcacheAlignment;
00567
Offset = (ULONG)BaseAddress & DcacheAlignment;
00568 HalSweepDcacheRange(
00569 (PVOID)((ULONG)BaseAddress & ~DcacheAlignment),
00570 (
Offset + Length + DcacheAlignment) & ~DcacheAlignment);
00571 }
00572
00573
#if 0
00574
00575
00576
00577
00578
00579
00580
00581
00582
00583
00584 OldIrql = KeRaiseIrqlToSynchLevel();
00585
00586
00587
00588
00589
00590
00591 TargetProcessors =
KeActiveProcessors & PCR->NotMember;
00592
if (TargetProcessors != 0) {
00593
KiIpiSendPacket(TargetProcessors,
00594
KiSweepIcacheRangeTarget,
00595 (PVOID)BaseAddress,
00596 (PVOID)Length,
00597
NULL);
00598 }
00599
00600
#endif
00601
00602
00603
00604
00605
00606
00607 IcacheAlignment = PCR->IcacheAlignment;
00608
Offset = (ULONG)BaseAddress & IcacheAlignment;
00609 HalSweepIcacheRange((PVOID)((ULONG)BaseAddress & ~IcacheAlignment),
00610 (
Offset + Length + IcacheAlignment) & ~IcacheAlignment);
00611
00612
00613
00614
00615
00616
00617
#if 0
00618
00619
if (TargetProcessors != 0) {
00620
KiIpiStallOnPacketTargets();
00621 }
00622
00623
00624
00625
00626
00627
KeLowerIrql(OldIrql);
00628
00629
#endif
00630
00631
return;
00632 }
00633
00634
VOID
00635
KiSweepIcacheRangeTarget (
00636 IN PULONG SignalDone,
00637 IN PVOID BaseAddress,
00638 IN PVOID Length,
00639 IN PVOID Parameter3
00640 )
00641
00642
00643
00644
00645
00646
00647
00648
00649
00650
00651
00652
00653
00654
00655
00656
00657
00658
00659
00660
00661
00662
00663
00664
00665
00666
00667
00668
00669
00670 {
00671
00672
#if 0
00673
00674 ULONG
Offset;
00675 ULONG IcacheAlignment;
00676
00677
00678
00679
00680 IcacheAlignment = PCR->IcacheAlignment;
00681
Offset = (ULONG)(BaseAddress) & IcacheAlignment;
00682 HalSweepIcacheRange((PVOID)((ULONG)(BaseAddress) & ~IcacheAlignment),
00683 (Offset + (ULONG)Length + IcacheAlignment) & ~IcacheAlignment);
00684
00685
KiIpiSignalPacketDone(SignalDone);
00686
00687
#endif
00688
00689
return;
00690 }
00691
00692
VOID
00693 KeFlushIoBuffers (
00694 IN
PMDL Mdl,
00695 IN BOOLEAN ReadOperation,
00696 IN BOOLEAN DmaOperation
00697 )
00698
00699
00700
00701
00702
00703
00704
00705
00706
00707
00708
00709
00710
00711
00712
00713
00714
00715
00716
00717
00718
00719
00720
00721
00722
00723 {
00724
00725 KIRQL OldIrql;
00726 KAFFINITY TargetProcessors;
00727 ULONG MaxLocalSweep;
00728
00729
ASSERT(KeGetCurrentIrql() <=
SYNCH_LEVEL);
00730
00731
00732
00733
00734
00735
00736
00737
00738
00739
if (DmaOperation !=
FALSE) {
00740
if (ReadOperation !=
FALSE) {
00741
00742
#if DBG
00743
00744
00745
00746
00747
00748
00749
00750
if ((
KiDmaIoCoherency &
DMA_READ_ICACHE_INVALIDATE) != 0) {
00751
00752
ASSERT((
KiDmaIoCoherency &
DMA_READ_DCACHE_INVALIDATE) != 0);
00753
00754
return;
00755 }
00756
00757
#endif
00758
00759
00760
00761
00762
00763
00764
00765
if ((Mdl->MdlFlags &
MDL_IO_PAGE_READ) == 0) {
00766
ASSERT((
KiDmaIoCoherency &
DMA_READ_DCACHE_INVALIDATE) != 0);
00767
return;
00768 }
00769
00770 }
else if ((
KiDmaIoCoherency &
DMA_WRITE_DCACHE_SNOOP) != 0) {
00771
return;
00772 }
00773
00774 }
else if ((Mdl->MdlFlags &
MDL_IO_PAGE_READ) == 0) {
00775
return;
00776 }
00777
00778
00779
00780
00781
00782
00783
00784
00785
if ((KeGetPvr() >> 16) == 1) {
00786
return;
00787 }
00788
00789
00790
00791
00792
00793
00794
00795
00796
00797
00798
00799
00800
00801
00802
00803
00804
00805
00806
00807
00808
00809
00810 MaxLocalSweep = PCR->FirstLevelDcacheSize;
00811
00812
#if !defined(NT_UP)
00813
00814 MaxLocalSweep *=
KeNumberProcessors;
00815
00816
#endif
00817
00818
if (Mdl->ByteCount > MaxLocalSweep) {
00819
00820
00821
00822
00823
00824 OldIrql = KeRaiseIrqlToSynchLevel();
00825
00826
#if !defined(NT_UP)
00827
00828
00829
00830
00831
00832
00833 TargetProcessors =
KeActiveProcessors & PCR->NotMember;
00834
if (TargetProcessors != 0) {
00835
00836
KiIpiSendPacket(TargetProcessors,
00837
KiFlushIoBuffersTarget,
00838 (PVOID)Mdl,
00839 (PVOID)((ULONG)ReadOperation),
00840 (PVOID)((ULONG)DmaOperation));
00841
00842 }
00843
00844
#endif
00845
00846
00847
00848
00849
00850 HalSweepDcache();
00851
00852 HalSweepIcache();
00853
00854
00855
00856
00857
00858
00859
#if !defined(NT_UP)
00860
00861
if (TargetProcessors != 0) {
00862
KiIpiStallOnPacketTargets();
00863 }
00864
00865
#endif
00866
00867
00868
00869
00870
00871
KeLowerIrql(OldIrql);
00872
00873
return;
00874 }
00875
00876
00877
00878
00879
00880
00881
00882 HalFlushIoBuffers(Mdl, ReadOperation, DmaOperation);
00883
00884 }
00885
00886
VOID
00887
KiFlushIoBuffersTarget (
00888 IN PULONG SignalDone,
00889 IN PVOID Mdl,
00890 IN PVOID ReadOperation,
00891 IN PVOID DmaOperation
00892 )
00893
00894
00895
00896
00897
00898
00899
00900
00901
00902
00903
00904
00905
00906
00907
00908
00909
00910
00911
00912
00913
00914
00915
00916
00917
00918
00919
00920
00921
00922
00923 {
00924
00925
00926
00927
00928
00929
#if !defined(NT_UP)
00930
00931 HalSweepDcache();
00932
00933 HalSweepIcache();
00934
00935
KiIpiSignalPacketDone(SignalDone);
00936
00937
#endif
00938
00939
return;
00940 }