diff --git a/android/abi_gki_aarch64.xml b/android/abi_gki_aarch64.xml old mode 100755 new mode 100644 index b3258f6a0bcb..e5b5a17b490b --- a/android/abi_gki_aarch64.xml +++ b/android/abi_gki_aarch64.xml @@ -85,6 +85,9 @@ + + + @@ -112,6 +115,14 @@ + + + + + + + + @@ -131,6 +142,8 @@ + + @@ -601,6 +614,7 @@ + @@ -626,6 +640,7 @@ + @@ -804,6 +819,7 @@ + @@ -881,6 +897,7 @@ + @@ -933,6 +950,7 @@ + @@ -1044,6 +1062,7 @@ + @@ -1311,11 +1330,15 @@ + + + + @@ -1361,6 +1384,7 @@ + @@ -1794,8 +1818,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2337,6 +2391,7 @@ + @@ -2374,6 +2429,7 @@ + @@ -2409,17 +2465,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -2464,8 +2564,11 @@ + + + @@ -2476,6 +2579,7 @@ + @@ -2560,10 +2664,12 @@ + + @@ -2591,9 +2697,14 @@ + + + + + @@ -2754,6 +2865,8 @@ + + @@ -2783,6 +2896,7 @@ + @@ -2807,6 +2921,7 @@ + @@ -2838,6 +2953,7 @@ + @@ -3987,6 +4103,9 @@ + + + @@ -4506,6 +4625,7 @@ + @@ -4553,6 +4673,7 @@ + @@ -4562,6 +4683,7 @@ + @@ -4599,6 +4721,7 @@ + @@ -4995,6 +5118,7 @@ + @@ -5064,6 +5188,8 @@ + + @@ -5692,7 +5818,10 @@ + + + @@ -5829,6 +5958,7 @@ + @@ -6260,6 +6390,7 @@ + @@ -6285,6 +6416,7 @@ + @@ -6454,6 +6586,7 @@ + @@ -7140,7 +7273,6 @@ - @@ -8916,9 +9048,9 @@ - - - + + + @@ -8939,7 +9071,6 @@ - @@ -9021,7 +9152,6 @@ - @@ -9711,6 +9841,7 @@ + @@ -10412,6 +10543,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -11897,6 +12063,7 @@ + @@ -12303,8 +12470,8 @@ - - + + @@ -12945,7 +13112,23 @@ - + + + + + + + + + + + + + + + + + @@ -13593,6 +13776,7 @@ + @@ -14002,6 +14186,7 @@ + @@ -14870,6 +15055,7 @@ + @@ -15142,12 +15328,6 @@ - - - - - - @@ -15642,6 +15822,9 @@ + + + @@ -15798,6 +15981,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -15841,7 +16044,29 @@ - + + + + + + + + + + + + + + + + + + + + + + + @@ -18731,7 +18956,6 @@ - @@ -19122,11 +19346,7 @@ - - - - - + @@ -20081,7 +20301,6 @@ - @@ -20279,7 +20498,6 @@ - @@ -20720,14 +20938,6 @@ - - - - - - - - @@ -21078,7 +21288,8 @@ - + + @@ -21810,6 +22021,7 @@ + @@ -22550,6 +22762,11 @@ + + + + + @@ -24733,7 +24950,7 @@ - + @@ -24772,7 +24989,6 @@ - @@ -24959,6 +25175,7 @@ + @@ -25263,7 +25480,23 @@ - + + + + + + + + + + + + + + + + + @@ -25926,7 +26159,7 @@ - + @@ -26082,7 +26315,7 @@ - + @@ -26303,7 +26536,6 @@ - @@ -26381,7 +26613,7 @@ - + @@ -26438,6 +26670,7 @@ + @@ -26794,6 +27027,7 @@ + @@ -27432,7 +27666,7 @@ - + @@ -28374,7 +28608,6 @@ - @@ -31238,9 +31471,6 @@ - - - @@ -31354,6 +31584,7 @@ + @@ -31862,7 +32093,20 @@ - + + + + + + + + + + + + + + @@ -31886,6 +32130,14 @@ + + + + + + + + @@ -32083,6 +32335,7 @@ + @@ -32226,9 +32479,9 @@ - - - + + + @@ -32719,6 +32972,7 @@ + @@ -33115,7 +33369,6 @@ - @@ -33370,7 +33623,68 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -33598,6 +33912,11 @@ + + + + + @@ -33781,6 +34100,9 @@ + + + @@ -33877,10 +34199,9 @@ - - + @@ -34292,6 +34613,7 @@ + @@ -34490,7 +34812,7 @@ - + @@ -34641,7 +34963,7 @@ - + @@ -36127,10 +36449,6 @@ - - - - @@ -36445,6 +36763,12 @@ + + + + + + @@ -36812,7 +37136,17 @@ - + + + + + + + + + + + @@ -38040,7 +38374,6 @@ - @@ -38831,8 +39164,8 @@ - - + + @@ -39805,17 +40138,6 @@ - - - - - - - - - - - @@ -40096,9 +40418,9 @@ - - - + + + @@ -40605,6 +40927,7 @@ + @@ -41588,7 +41911,6 @@ - @@ -44061,13 +44383,6 @@ - - - - - - - @@ -44097,11 +44412,6 @@ - - - - - @@ -44549,12 +44859,12 @@ - - - - - - + + + + + + @@ -44659,7 +44969,6 @@ - @@ -44755,6 +45064,7 @@ + @@ -45440,21 +45750,21 @@ - + - + - + - + - + - + @@ -47193,6 +47503,12 @@ + + + + + + @@ -47553,8 +47869,8 @@ - - + + @@ -47900,7 +48216,35 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -48016,6 +48360,7 @@ + @@ -48081,6 +48426,7 @@ + @@ -48114,7 +48460,7 @@ - + @@ -48465,7 +48811,14 @@ - + + + + + + + + @@ -49111,11 +49464,6 @@ - - - - - @@ -49527,7 +49875,7 @@ - + @@ -49653,6 +50001,11 @@ + + + + + @@ -50170,56 +50523,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -50560,6 +50864,7 @@ + @@ -51507,7 +51812,7 @@ - + @@ -52352,7 +52657,6 @@ - @@ -52787,7 +53091,6 @@ - @@ -52838,6 +53141,7 @@ + @@ -53448,7 +53752,23 @@ - + + + + + + + + + + + + + + + + + @@ -53810,7 +54130,6 @@ - @@ -54741,9 +55060,9 @@ - - - + + + @@ -55463,6 +55782,7 @@ + @@ -56047,7 +56367,7 @@ - + @@ -56151,6 +56471,7 @@ + @@ -56512,7 +56833,7 @@ - + @@ -57612,8 +57933,10 @@ + + @@ -58750,7 +59073,6 @@ - @@ -59213,11 +59535,6 @@ - - - - - @@ -60442,44 +60759,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -61705,65 +61985,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -61795,6 +62017,7 @@ + @@ -63181,7 +63404,6 @@ - @@ -63486,7 +63708,17 @@ - + + + + + + + + + + + @@ -64566,7 +64798,7 @@ - + @@ -65060,10 +65292,10 @@ - - - - + + + + @@ -65114,6 +65346,11 @@ + + + + + @@ -66333,11 +66570,6 @@ - - - - - @@ -66588,6 +66820,7 @@ + @@ -67372,11 +67605,6 @@ - - - - - @@ -67838,7 +68066,7 @@ - + @@ -69901,6 +70129,7 @@ + @@ -70611,6 +70840,7 @@ + @@ -70778,6 +71008,7 @@ + @@ -72346,9 +72577,10 @@ + - - + + @@ -72467,7 +72699,6 @@ - @@ -72792,7 +73023,6 @@ - @@ -73109,6 +73339,7 @@ + @@ -73156,8 +73387,8 @@ - - + + @@ -73229,6 +73460,7 @@ + @@ -73696,38 +73928,7 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -73838,7 +74039,7 @@ - + @@ -75393,6 +75594,7 @@ + @@ -76529,7 +76731,6 @@ - @@ -77937,12 +78138,12 @@ - + - + @@ -78998,12 +79199,12 @@ - - - - - - + + + + + + @@ -80300,6 +80501,7 @@ + @@ -80539,7 +80741,6 @@ - @@ -81265,6 +81466,7 @@ + @@ -81317,6 +81519,23 @@ + + + + + + + + + + + + + + + + + @@ -81852,6 +82071,7 @@ + @@ -81924,6 +82144,9 @@ + + + @@ -81940,11 +82163,6 @@ - - - - - @@ -81967,7 +82185,6 @@ - @@ -82428,7 +82645,7 @@ - + @@ -82504,14 +82721,6 @@ - - - - - - - - @@ -85619,6 +85828,7 @@ + @@ -86237,7 +86447,7 @@ - + @@ -86304,6 +86514,7 @@ + @@ -86918,9 +87129,9 @@ - - - + + + @@ -87389,26 +87600,7 @@ - - - - - - - - - - - - - - - - - - - - + @@ -87433,7 +87625,7 @@ - + @@ -87458,6 +87650,7 @@ + @@ -87496,6 +87689,7 @@ + @@ -88910,7 +89104,7 @@ - + @@ -89557,8 +89751,8 @@ - - + + @@ -90794,6 +90988,14 @@ + + + + + + + + @@ -91228,7 +91430,20 @@ - + + + + + + + + + + + + + + @@ -91785,7 +92000,7 @@ - + @@ -91960,7 +92175,7 @@ - + @@ -92255,6 +92470,7 @@ + @@ -92395,12 +92611,12 @@ - + - + @@ -92492,7 +92708,6 @@ - @@ -93967,7 +94182,6 @@ - @@ -95253,7 +95467,7 @@ - + @@ -95765,7 +95979,7 @@ - + @@ -95811,7 +96025,7 @@ - + @@ -96249,7 +96463,7 @@ - + @@ -96503,6 +96717,7 @@ + @@ -96956,9 +97171,6 @@ - - - @@ -98198,6 +98410,7 @@ + @@ -98473,8 +98686,8 @@ - - + + @@ -98684,8 +98897,8 @@ - - + + @@ -99842,6 +100055,7 @@ + @@ -100707,6 +100921,7 @@ + @@ -100819,7 +101034,14 @@ - + + + + + + + + @@ -102304,9 +102526,6 @@ - - - @@ -102319,6 +102538,9 @@ + + + @@ -102342,7 +102564,7 @@ - + @@ -102483,8 +102705,8 @@ - - + + @@ -102755,6 +102977,7 @@ + @@ -103084,7 +103307,6 @@ - @@ -103353,6 +103575,7 @@ + @@ -103887,7 +104110,20 @@ - + + + + + + + + + + + + + + @@ -103946,6 +104182,9 @@ + + + @@ -107390,6 +107629,7 @@ + @@ -107404,20 +107644,6 @@ - - - - - - - - - - - - - - @@ -107752,6 +107978,7 @@ + @@ -109841,6 +110068,7 @@ + @@ -111687,11 +111915,6 @@ - - - - - @@ -112018,6 +112241,7 @@ + @@ -113304,6 +113528,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -113785,7 +114029,6 @@ - @@ -113870,7 +114113,6 @@ - @@ -113994,8 +114236,8 @@ - - + + @@ -114510,6 +114752,23 @@ + + + + + + + + + + + + + + + + + @@ -114576,6 +114835,7 @@ + @@ -114638,6 +114898,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -114758,6 +115063,28 @@ + + + + + + + + + + + + + + + + + + + + + + @@ -117659,6 +117986,13 @@ + + + + + + + @@ -117807,6 +118141,13 @@ + + + + + + + @@ -118194,6 +118535,7 @@ + @@ -118219,6 +118561,7 @@ + @@ -118415,9 +118758,9 @@ - - - + + + @@ -118428,33 +118771,33 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + @@ -118464,27 +118807,27 @@ - - - + + + - - - + + + - - - + + + - - - + + + @@ -118494,8 +118837,8 @@ - - + + @@ -118514,8 +118857,8 @@ - - + + @@ -118531,57 +118874,57 @@ - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -118592,16 +118935,16 @@ - - + + - - + + @@ -118625,12 +118968,12 @@ - - - - - - + + + + + + @@ -118647,9 +118990,9 @@ - - - + + + @@ -118665,29 +119008,29 @@ - - + + - - - - - + + + + + - - - + + + - - + + - - - + + + @@ -118704,15 +119047,15 @@ - - - + + + - - - + + + @@ -118725,22 +119068,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -118819,13 +119162,13 @@ - - - - - - - + + + + + + + @@ -118834,29 +119177,29 @@ - - - + + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -118898,11 +119241,11 @@ - - - - - + + + + + @@ -118922,14 +119265,14 @@ - - + + - - - - + + + + @@ -118952,15 +119295,15 @@ - - - - + + + + - - - + + + @@ -118968,11 +119311,11 @@ - - - - - + + + + + @@ -119023,21 +119366,21 @@ - - - - - + + + + + - - - - - - + + + + + + - - + + @@ -119078,18 +119421,18 @@ - - + + - - - + + + - - - + + + @@ -119122,17 +119465,17 @@ - - - - + + + + - - - - - + + + + + @@ -119143,42 +119486,47 @@ - - - - + + + + - - + + - - - + + + - - - + + + - - + + + - - - - + + - - + + + + - - + + + + + + @@ -119188,15 +119536,15 @@ - - + + - - - - - + + + + + @@ -119209,9 +119557,9 @@ - - - + + + @@ -119220,49 +119568,49 @@ - - - - + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -119272,10 +119620,10 @@ - - - - + + + + @@ -119325,17 +119673,17 @@ - - + + - - + + - - - + + + @@ -119370,10 +119718,10 @@ - - - - + + + + @@ -119405,13 +119753,13 @@ - - - + + + - - + + @@ -119442,8 +119790,8 @@ - - + + @@ -119452,8 +119800,8 @@ - - + + @@ -119475,8 +119823,8 @@ - - + + @@ -119523,8 +119871,8 @@ - - + + @@ -119546,10 +119894,14 @@ - - - - + + + + + + + + @@ -119585,9 +119937,9 @@ - - - + + + @@ -119736,16 +120088,16 @@ - - + + - - + + @@ -119794,34 +120146,42 @@ - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - - + + + + + + + + + + + @@ -119842,31 +120202,31 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -119884,9 +120244,9 @@ - - - + + + @@ -119934,67 +120294,67 @@ - - - - + + + + - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -120010,16 +120370,16 @@ - - + + - - - - - - + + + + + + @@ -120040,22 +120400,22 @@ - - + + - - + + - - + + - - + + @@ -120079,23 +120439,23 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -120105,8 +120465,8 @@ - - + + @@ -120114,13 +120474,13 @@ - - - + + + - - + + @@ -120190,9 +120550,9 @@ - - - + + + @@ -120236,11 +120596,11 @@ - - - - - + + + + + @@ -120256,19 +120616,19 @@ - - - + + + - - - + + + - - - + + + @@ -120300,9 +120660,9 @@ - - - + + + @@ -120391,14 +120751,14 @@ - - - + + + - - - + + + @@ -120407,47 +120767,47 @@ - - - + + + - - - - + + + + - - + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - + + + + - - + + @@ -120460,26 +120820,30 @@ - - + + - - + + + + + + - - - + + + - - - + + + @@ -120488,48 +120852,48 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -120540,8 +120904,8 @@ - - + + @@ -120659,8 +121023,8 @@ - - + + @@ -120768,28 +121132,28 @@ - - - + + + - - + + - - - - - + + + + + @@ -120800,13 +121164,13 @@ - - - - - - - + + + + + + + @@ -120817,11 +121181,11 @@ - - - - - + + + + + @@ -120833,11 +121197,11 @@ - - - - - + + + + + @@ -120856,8 +121220,8 @@ - - + + @@ -120894,10 +121258,10 @@ - - - - + + + + @@ -120935,22 +121299,22 @@ - - + + - - - - + + + + - - + + - - + + @@ -120961,9 +121325,9 @@ - - - + + + @@ -120982,22 +121346,22 @@ - - + + - - - - + + + + - - - + + + @@ -121058,8 +121422,8 @@ - - + + @@ -121090,18 +121454,18 @@ - - + + - - - + + + - - - + + + @@ -121113,8 +121477,8 @@ - - + + @@ -121130,9 +121494,9 @@ - - - + + + @@ -121165,12 +121529,12 @@ - - + + - - + + @@ -121181,23 +121545,23 @@ - - + + - - - + + + - - - - + + + + @@ -121212,9 +121576,9 @@ - - - + + + @@ -121233,8 +121597,8 @@ - - + + @@ -121254,58 +121618,58 @@ - - - + + + - - - + + + - - - - + + + + - - - - - + + + + + - - - - - + + + + + - + - + - - - + + + - - - - - + + + + + - - - - - + + + + + @@ -121354,12 +121718,12 @@ - - + + - - + + @@ -121388,10 +121752,10 @@ - - - - + + + + @@ -121446,42 +121810,42 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - + + + - - - + + + - + @@ -121524,10 +121888,10 @@ - - - - + + + + @@ -121535,9 +121899,9 @@ - - - + + + @@ -121547,7 +121911,7 @@ - + @@ -121568,8 +121932,8 @@ - - + + @@ -121590,22 +121954,22 @@ - - + + - - - + + + - - - + + + @@ -121655,11 +122019,11 @@ - - - - - + + + + + @@ -121667,44 +122031,44 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -121725,8 +122089,8 @@ - - + + @@ -121751,18 +122115,18 @@ - - - + + + - - - + + + @@ -121779,9 +122143,9 @@ - - - + + + @@ -121792,11 +122156,11 @@ - - - - - + + + + + @@ -121806,11 +122170,11 @@ - - + + - + @@ -121832,27 +122196,40 @@ - - + + - - - + + + + + + + - - + + + + + + + + + + + - - - - + + + + @@ -121860,6 +122237,11 @@ + + + + + @@ -121888,12 +122270,12 @@ - - + + - - + + @@ -121902,7 +122284,7 @@ - + @@ -121933,11 +122315,11 @@ - - - - - + + + + + @@ -121947,25 +122329,25 @@ - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -122002,11 +122384,11 @@ - - - - - + + + + + @@ -122016,11 +122398,11 @@ - - - - - + + + + + @@ -122052,9 +122434,9 @@ - - - + + + @@ -122065,8 +122447,8 @@ - - + + @@ -122076,9 +122458,9 @@ - - - + + + @@ -122087,11 +122469,11 @@ - - - - - + + + + + @@ -122104,16 +122486,16 @@ - - + + - - + + - - + + @@ -122124,12 +122506,18 @@ + + + + + + - - + + @@ -122171,49 +122559,49 @@ - - + + - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -122235,9 +122623,9 @@ - - - + + + @@ -122250,9 +122638,9 @@ - - - + + + @@ -122274,10 +122662,10 @@ - - - - + + + + @@ -122308,15 +122696,15 @@ - - - - + + + + - - - + + + @@ -122336,8 +122724,8 @@ - - + + @@ -122353,8 +122741,8 @@ - - + + @@ -122362,13 +122750,13 @@ - - + + - - - + + + @@ -122379,9 +122767,9 @@ - - - + + + @@ -122398,16 +122786,16 @@ - - + + - - + + @@ -122432,9 +122820,9 @@ - - - + + + @@ -122524,9 +122912,9 @@ - - - + + + @@ -122539,17 +122927,17 @@ - - - - + + + + - - - - + + + + @@ -122557,10 +122945,10 @@ - - - - + + + + @@ -122568,9 +122956,9 @@ - - - + + + @@ -122595,11 +122983,11 @@ - - - - - + + + + + @@ -122671,20 +123059,20 @@ - - + + - - + + - - + + @@ -122697,8 +123085,8 @@ - - + + @@ -122720,12 +123108,12 @@ - - - - - - + + + + + + @@ -122734,40 +123122,40 @@ - - - + + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -122802,28 +123190,28 @@ - - - + + + - - + + - - - - + + + + - - - + + + @@ -122845,10 +123233,10 @@ - - - - + + + + @@ -122870,11 +123258,11 @@ - - - - - + + + + + @@ -122897,15 +123285,15 @@ - - - + + + - - - - + + + + @@ -122922,14 +123310,14 @@ - - - + + + - - - + + + @@ -122958,22 +123346,22 @@ - - + + - - + + - - - - + + + + @@ -122996,10 +123384,10 @@ - - - - + + + + @@ -123007,15 +123395,15 @@ - - - - + + + + - - - + + + @@ -123122,11 +123510,11 @@ - - - - - + + + + + @@ -123324,29 +123712,29 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -123356,10 +123744,10 @@ - - - - + + + + @@ -123398,14 +123786,14 @@ - - - - - - - - + + + + + + + + @@ -123418,9 +123806,9 @@ - - - + + + @@ -123453,8 +123841,8 @@ - - + + @@ -123521,14 +123909,14 @@ - - - + + + - - - + + + @@ -123536,10 +123924,10 @@ - - - - + + + + @@ -123584,15 +123972,15 @@ - - - - + + + + - - - + + + @@ -123661,17 +124049,17 @@ - - - + + + - - - - - - + + + + + + @@ -123691,16 +124079,16 @@ - - - + + + - - - - - + + + + + @@ -123756,46 +124144,46 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - + + - - - - - + + + + + - - + + - - + + @@ -123904,9 +124292,9 @@ - - - + + + @@ -123928,17 +124316,17 @@ - - + + - - - + + + - - + + @@ -123946,15 +124334,15 @@ - - - + + + - - - - + + + + @@ -123965,34 +124353,34 @@ - - + + - - - - + + + + - - + + - - - + + + - - - - + + + + @@ -124004,9 +124392,9 @@ - - - + + + @@ -124014,8 +124402,8 @@ - - + + @@ -124026,15 +124414,15 @@ - + - - - - - - + + + + + + @@ -124046,13 +124434,13 @@ - - - - + + + + - - + + @@ -124072,10 +124460,10 @@ - - - - + + + + @@ -124233,46 +124621,46 @@ - - - + + + - - - + + + - - + + - - - - - + + + + + - - + + - - - + + + - - - - + + + + - - - - - + + + + + @@ -124405,8 +124793,8 @@ - - + + @@ -124439,16 +124827,16 @@ - - + + - - + + - - + + @@ -124460,8 +124848,8 @@ - - + + @@ -124472,10 +124860,15 @@ - - + + + + + + + @@ -124485,6 +124878,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -124493,9 +125031,9 @@ - - - + + + @@ -124505,62 +125043,62 @@ - - + + - - - + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -124578,14 +125116,14 @@ - - - + + + - - - + + + @@ -124620,13 +125158,13 @@ - - - - - - - + + + + + + + @@ -124850,62 +125388,62 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + @@ -124921,15 +125459,15 @@ - - + + - - - - - + + + + + @@ -124940,10 +125478,10 @@ - - - - + + + + @@ -124972,8 +125510,8 @@ - - + + @@ -124981,9 +125519,9 @@ - - - + + + @@ -125020,9 +125558,9 @@ - - - + + + @@ -125031,8 +125569,8 @@ - - + + @@ -125040,19 +125578,19 @@ - - - - - + + + + + - - - - - - + + + + + + @@ -125060,20 +125598,20 @@ - - + + - - + + - - + + - - + + @@ -125086,17 +125624,17 @@ - - + + - - + + - - - + + + @@ -125143,19 +125681,19 @@ - - - - - - - + + + + + + + - - - + + + @@ -125163,17 +125701,17 @@ - - - + + + - - + + @@ -125191,20 +125729,20 @@ - - + + - - + + - - + + @@ -125217,11 +125755,11 @@ - - - - - + + + + + @@ -125232,33 +125770,33 @@ - - - + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -125542,23 +126080,23 @@ - - + + - - - - + + + + - - - + + + @@ -125574,12 +126112,12 @@ - - - - - - + + + + + + @@ -125631,20 +126169,20 @@ - - + + - - - - + + + + - - - - + + + + @@ -125799,10 +126337,10 @@ - - - - + + + + @@ -125852,9 +126390,9 @@ - - - + + + @@ -125983,19 +126521,19 @@ - - - + + + - - - + + + - - - + + + @@ -126096,8 +126634,8 @@ - - + + @@ -126119,9 +126657,9 @@ - - - + + + @@ -126149,8 +126687,8 @@ - - + + @@ -126165,24 +126703,24 @@ - - + + - - - + + + - - - - + + + + - - - + + + @@ -126201,9 +126739,9 @@ - - - + + + @@ -126217,18 +126755,18 @@ - - + + - - - + + + - - - + + + @@ -126249,9 +126787,9 @@ - - - + + + @@ -126282,28 +126820,28 @@ - - + + - - + + - - - + + + - - - - + + + + - - - + + + @@ -126314,83 +126852,83 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - + + - - + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + @@ -126446,17 +126984,17 @@ - - + + - - + + - - - + + + @@ -126470,20 +127008,20 @@ - - + + - - + + - - + + @@ -126552,9 +127090,9 @@ - - - + + + @@ -126562,44 +127100,44 @@ - - + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + @@ -126609,12 +127147,12 @@ - - - - - - + + + + + + @@ -126625,9 +127163,9 @@ - - - + + + @@ -126635,18 +127173,18 @@ - - - - + + + + - - - - - - - + + + + + + + @@ -126661,30 +127199,30 @@ - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -126720,8 +127258,8 @@ - - + + @@ -126797,20 +127335,20 @@ - - + + - - - - - - - - - - + + + + + + + + + + @@ -126819,19 +127357,19 @@ - - - + + + - - - - + + + + - - - + + + @@ -126858,40 +127396,40 @@ - - - - + + + + - - + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -127130,8 +127668,8 @@ - - + + @@ -127145,13 +127683,13 @@ - - + + - - - + + + @@ -127168,10 +127706,10 @@ - - - - + + + + @@ -127204,14 +127742,14 @@ - - - + + + - - - + + + @@ -127219,17 +127757,17 @@ - - - - + + + + - - + + @@ -127244,9 +127782,9 @@ - - - + + + @@ -127260,50 +127798,54 @@ - - - - + + + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + + + + + - - - + + + - - + + @@ -127313,15 +127855,15 @@ - - - + + + - - - - + + + + @@ -127354,18 +127896,18 @@ - - - - + + + + - - - - - + + + + + @@ -127373,13 +127915,13 @@ - - - + + + - - + + @@ -127409,12 +127951,12 @@ - - + + - - + + @@ -127460,15 +128002,15 @@ - - - + + + - - - - + + + + @@ -127477,30 +128019,37 @@ - - - + + + - - + + - - - + + + + + + + + + + - - - - + + + + - - - - + + + + @@ -127516,26 +128065,26 @@ - - + + - - - + + + - - + + - - - + + + @@ -127552,11 +128101,11 @@ - - + + - + @@ -127571,8 +128120,8 @@ - - + + @@ -127620,18 +128169,18 @@ - - - + + + - - + + - - - + + + @@ -127639,23 +128188,31 @@ - - + + - - - + + + + + + + + + + + - - - + + + @@ -127687,6 +128244,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -127715,12 +128452,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -127729,9 +128507,9 @@ - - - + + + @@ -127744,8 +128522,8 @@ - - + + @@ -127788,12 +128566,12 @@ - - + + - - + + @@ -127830,35 +128608,35 @@ - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + @@ -127870,13 +128648,13 @@ - - - + + + - - + + @@ -127919,44 +128697,44 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - + + + - - - - + + + + - - + + - - + + @@ -127964,6 +128742,22 @@ + + + + + + + + + + + + + + + + @@ -127975,6 +128769,14 @@ + + + + + + + + @@ -127998,18 +128800,18 @@ - - - + + + - - + + - - - + + + @@ -128032,14 +128834,18 @@ + + + + - - - - + + + + @@ -128049,12 +128855,12 @@ - - + + - - + + @@ -128065,17 +128871,17 @@ - - - - - - - + + + + + + + - - + + @@ -128092,8 +128898,8 @@ - - + + @@ -128123,15 +128929,15 @@ - - + + - - - - - + + + + + @@ -128143,8 +128949,8 @@ - - + + @@ -128170,8 +128976,8 @@ - - + + @@ -128181,20 +128987,20 @@ - - + + - + - - - + + + @@ -128208,41 +129014,41 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -128252,13 +129058,13 @@ - - + + - - - + + + @@ -128272,41 +129078,41 @@ - - - - - - + + + + + + - - + + - - - - - - + + + + + + - - - - - + + + + + - - - - - - - - + + + + + + + + @@ -128320,8 +129126,8 @@ - - + + @@ -128333,18 +129139,18 @@ - - + + - - - - + + + + @@ -128368,23 +129174,23 @@ - - - + + + - - - + + + - - - + + + - - + + @@ -128397,9 +129203,9 @@ - - - + + + @@ -128407,29 +129213,29 @@ - - - - + + + + - - + + - - - - - + + + + + - - - - - - + + + + + + @@ -128457,14 +129263,25 @@ + + + + + + + + + + + - - - + + + @@ -128473,13 +129290,19 @@ - - + + - - - + + + + + + + + + @@ -128576,8 +129399,8 @@ - - + + @@ -128604,17 +129427,40 @@ + + + + + + + + + + + - - - + + + - - + + + + + + + + + + + + + + @@ -128659,10 +129505,10 @@ - - + + - + @@ -128677,22 +129523,22 @@ - - - + + + - - - + + + - - + + - - + + @@ -128743,38 +129589,38 @@ - - - - + + + + - - - - - - - - + + + + + + + + - - + + - - + + - + - - + + @@ -128785,9 +129631,9 @@ - - - + + + @@ -128802,26 +129648,26 @@ - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -128836,28 +129682,28 @@ - - + + - - - + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128872,28 +129718,28 @@ - - - - - + + + + + - - + + - - - - - + + + + + - - - - + + + + @@ -128915,18 +129761,18 @@ - - + + - - + + - - + + @@ -128949,12 +129795,12 @@ - - + + - - + + @@ -128991,9 +129837,9 @@ - - - + + + @@ -129028,81 +129874,81 @@ - - - + + + - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -129232,12 +130078,12 @@ - - + + - - + + @@ -129280,14 +130126,14 @@ - - + + - - - - + + + + @@ -129308,36 +130154,36 @@ - - - - - + + + + + - - + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + @@ -129348,39 +130194,39 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - - - + + + + @@ -129406,14 +130252,14 @@ - - + + - - - - + + + + @@ -129424,13 +130270,22 @@ - - - + + + + + + + - - + + + + + + + @@ -129446,8 +130301,8 @@ - - + + @@ -129569,21 +130424,26 @@ - - - - - + + + - - - - - - - - + + + + + + + + + + + + + + + @@ -129593,8 +130453,8 @@ - - + + @@ -129617,8 +130477,8 @@ - - + + @@ -129626,9 +130486,9 @@ - - - + + + @@ -129648,24 +130508,24 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -129673,46 +130533,50 @@ - - - - - - + + + + + + - - - + + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - + + + + + + + @@ -129728,8 +130592,8 @@ - - + + @@ -129750,8 +130614,8 @@ - - + + @@ -129766,8 +130630,8 @@ - - + + @@ -129800,20 +130664,20 @@ - - + + - - + + - - + + @@ -129850,47 +130714,51 @@ - - + + - - + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -129907,56 +130775,56 @@ - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -129975,9 +130843,9 @@ - - - + + + @@ -129999,42 +130867,42 @@ - - + + - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -130042,170 +130910,170 @@ - + - - - - + + + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - - + + + - - - - + + + + - - - - + + + + @@ -130213,53 +131081,53 @@ - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - - - - + + + + + @@ -130294,8 +131162,8 @@ - - + + @@ -130395,13 +131263,13 @@ - - - + + + - - + + @@ -130412,15 +131280,15 @@ - - - - - - - - - + + + + + + + + + @@ -130456,9 +131324,9 @@ - - - + + + @@ -130484,30 +131352,30 @@ - - - + + + - - - - + + + + - - - - + + + + - - - + + + @@ -130520,10 +131388,10 @@ - - - - + + + + @@ -130531,11 +131399,11 @@ - - - - - + + + + + @@ -130558,10 +131426,10 @@ - - - - + + + + @@ -130574,20 +131442,20 @@ - - - + + + - - - - + + + + - + - - + + @@ -130601,9 +131469,9 @@ - - - + + + @@ -130705,14 +131573,14 @@ - - - + + + - - + + @@ -130799,8 +131667,8 @@ - - + + @@ -130810,56 +131678,56 @@ - - + + - - - - - - - - - + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - + + - - + + - - + + - - + + - + @@ -130870,10 +131738,10 @@ - - - - + + + + @@ -130899,21 +131767,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -130924,35 +131792,35 @@ - - - - - - - - + + + + + + + + - - + + - - - + + + - - - - + + + + - - - - + + + + @@ -130984,46 +131852,46 @@ - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - - - + + + + + - - + + - - - + + + @@ -131032,35 +131900,35 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + - - + + @@ -131069,23 +131937,23 @@ - - - + + + - - - + + + - - - + + + @@ -131093,34 +131961,34 @@ - - - - + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + @@ -131130,16 +131998,16 @@ - - - - + + + + - - - - + + + + @@ -131155,10 +132023,10 @@ - - - - + + + + @@ -131168,10 +132036,10 @@ - - - - + + + + @@ -131181,37 +132049,37 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -131222,9 +132090,9 @@ - - - + + + @@ -131238,11 +132106,11 @@ - - - - - + + + + + @@ -131278,11 +132146,11 @@ - - + + - + @@ -131299,15 +132167,15 @@ - - + + - - + + - - + + @@ -131327,8 +132195,8 @@ - - + + @@ -131338,8 +132206,8 @@ - - + + @@ -131351,8 +132219,8 @@ - - + + @@ -131366,18 +132234,18 @@ - - - + + + - - - - + + + + - - + + @@ -131385,10 +132253,10 @@ - - - - + + + + @@ -131401,14 +132269,14 @@ - - - - + + + + - - + + @@ -131444,18 +132312,18 @@ - - - + + + - - - + + + @@ -131466,8 +132334,8 @@ - - + + @@ -131478,10 +132346,10 @@ - - - - + + + + @@ -131491,30 +132359,30 @@ - - - - - + + + + + - - + + - - + + - - - + + + - - + + @@ -131533,11 +132401,11 @@ - - - - - + + + + + @@ -131545,13 +132413,13 @@ - - + + - - + + @@ -131568,11 +132436,11 @@ - - - - - + + + + + @@ -131611,8 +132479,8 @@ - - + + @@ -131620,25 +132488,25 @@ - - - + + + - - + + - - - - + + + + - - - - + + + + @@ -131651,9 +132519,9 @@ - - - + + + @@ -131664,13 +132532,13 @@ - - + + - - - + + + @@ -131678,50 +132546,50 @@ - - - + + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + @@ -131735,8 +132603,8 @@ - - + + @@ -131812,9 +132680,9 @@ - - - + + + @@ -131852,32 +132720,32 @@ - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - + @@ -131890,9 +132758,9 @@ - - - + + + @@ -131972,10 +132840,10 @@ - - - - + + + + @@ -132020,13 +132888,13 @@ - - - - - - - + + + + + + + @@ -132213,12 +133081,12 @@ - - + + - - + + @@ -132230,11 +133098,11 @@ - - - - - + + + + + @@ -132275,8 +133143,8 @@ - - + + @@ -132329,8 +133197,8 @@ - - + + @@ -132342,9 +133210,9 @@ - - - + + + @@ -132355,28 +133223,28 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - + + + + + @@ -132387,9 +133255,9 @@ - - - + + + @@ -132450,8 +133318,8 @@ - - + + @@ -132490,9 +133358,9 @@ - - - + + + @@ -132548,8 +133416,8 @@ - - + + @@ -132571,12 +133439,12 @@ - - + + - - + + @@ -132610,8 +133478,8 @@ - - + + @@ -132622,9 +133490,9 @@ - - - + + + @@ -132632,47 +133500,47 @@ - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - - - - + + + + + @@ -132680,18 +133548,18 @@ - - - - + + + + - - + + @@ -132699,14 +133567,14 @@ - - - - + + + + - - - + + + @@ -132742,17 +133610,17 @@ - - + + - - - + + + @@ -132762,9 +133630,9 @@ - - - + + + @@ -132806,12 +133674,12 @@ - - + + - - + + @@ -132825,67 +133693,67 @@ - - - + + + - - + + - - + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + @@ -132893,19 +133761,19 @@ - - + + - - + + - - - - - + + + + + @@ -132916,41 +133784,41 @@ - - - - + + + + - - - + + + - - + + - - - - - + + + + + - - + + - - + + @@ -132964,16 +133832,16 @@ - - - - + + + + - - - - + + + + @@ -132994,11 +133862,11 @@ - - - - - + + + + + @@ -133015,24 +133883,24 @@ - - - - + + + + - - - - - + + + + + - - - - - + + + + + @@ -133057,10 +133925,10 @@ - - - - + + + + @@ -133073,12 +133941,12 @@ - - + + - - + + @@ -133103,18 +133971,18 @@ - - - + + + - - + + - - - + + + @@ -133138,18 +134006,18 @@ - - - - - + + + + + - - - - - + + + + + @@ -133160,19 +134028,19 @@ - - - - + + + + - - + + - - - + + + @@ -133191,14 +134059,14 @@ - - - + + + - - - + + + @@ -133224,9 +134092,9 @@ - - - + + + @@ -133239,22 +134107,22 @@ - - - - + + + + - - + + - - + + @@ -133268,43 +134136,43 @@ - - + + - - - + + + - - - + + + - - - - - + + + + + - - - + + + - - - - - + + + + + @@ -133317,9 +134185,9 @@ - - - + + + @@ -133352,14 +134220,14 @@ - - - - + + + + - - + + @@ -133370,44 +134238,44 @@ - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -133441,16 +134309,16 @@ - - - - - - + + + + + + - - - + + + @@ -133458,9 +134326,9 @@ - - - + + + @@ -133504,33 +134372,33 @@ - - - + + + - - - + + + - - + + - - + + - - - + + + - - - - + + + + @@ -133555,9 +134423,9 @@ - - - + + + @@ -133573,32 +134441,32 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + - - - + + + @@ -133607,9 +134475,9 @@ - - - + + + @@ -133617,74 +134485,74 @@ - - - + + + - - + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - + + + + @@ -133696,42 +134564,42 @@ - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - + + @@ -133746,8 +134614,8 @@ - - + + @@ -133767,8 +134635,8 @@ - - + + @@ -133782,61 +134650,61 @@ - - - - - + + + + + - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - + + + + @@ -133854,12 +134722,12 @@ - - - - - - + + + + + + @@ -133870,12 +134738,12 @@ - - - - - - + + + + + + @@ -133890,9 +134758,9 @@ - - - + + + @@ -133905,15 +134773,15 @@ - - + + - - - - - + + + + + @@ -133930,10 +134798,10 @@ - - - - + + + + @@ -133953,10 +134821,10 @@ - - - - + + + + @@ -134002,10 +134870,10 @@ - - - - + + + + @@ -134020,16 +134888,16 @@ - - - + + + - - + + - - + + @@ -134052,11 +134920,11 @@ - - - - - + + + + + @@ -134086,16 +134954,16 @@ - - - - - - + + + + + + - - + + @@ -134198,33 +135066,33 @@ - - + + - - + + - - - - - - + + + + + + - - - + + + - - + + @@ -134250,12 +135118,12 @@ - - + + - - + + @@ -134266,16 +135134,16 @@ - - + + - - + + - - + + @@ -134360,19 +135228,19 @@ - - - + + + - - - + + + - - - + + + @@ -134387,8 +135255,8 @@ - - + + @@ -134419,9 +135287,9 @@ - - - + + + @@ -134451,14 +135319,14 @@ - - - + + + - - - + + + @@ -134471,9 +135339,9 @@ - - - + + + @@ -134481,10 +135349,10 @@ - - - - + + + + @@ -134498,12 +135366,12 @@ - - + + - - + + @@ -134524,9 +135392,9 @@ - - - + + + @@ -134538,9 +135406,9 @@ - - - + + + @@ -134588,8 +135456,8 @@ - - + + @@ -134635,9 +135503,9 @@ - - - + + + @@ -134653,10 +135521,10 @@ - - - - + + + + @@ -134677,10 +135545,10 @@ - - - - + + + + @@ -134713,17 +135581,17 @@ - - + + - - - + + + @@ -134737,55 +135605,55 @@ - - - - + + + + - - + + - - - + + + - - + + - - + + - - - - - + + + + + - - - + + + - - - + + + - - + + - - + + - - + + @@ -134807,20 +135675,20 @@ - - - - - - + + + + + + - - + + - - + + @@ -134828,21 +135696,21 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -134853,7 +135721,7 @@ - + @@ -134872,21 +135740,21 @@ - - - - + + + + - - - - - - - - - + + + + + + + + + @@ -134920,8 +135788,8 @@ - - + + @@ -134941,9 +135809,9 @@ - - - + + + @@ -134951,23 +135819,23 @@ - - - + + + - - - + + + - - - + + + @@ -134988,37 +135856,37 @@ - - + + - - - - + + + + - - - + + + - - + + - - - - + + + + @@ -135032,11 +135900,11 @@ - - - - - + + + + + @@ -135046,12 +135914,12 @@ - - - - - - + + + + + + @@ -135095,10 +135963,10 @@ - - - - + + + + @@ -135157,24 +136025,24 @@ - - + + - - + + - - + + @@ -135184,11 +136052,11 @@ - - - - - + + + + + @@ -135233,14 +136101,14 @@ - - - - + + + + - - - + + + @@ -135473,8 +136341,8 @@ - - + + @@ -135494,29 +136362,29 @@ - - - + + + - - + + - - + + - - + + - - + + - - + + @@ -135528,14 +136396,14 @@ - - - + + + - - - + + + @@ -135548,16 +136416,16 @@ - - - - + + + + - - - - + + + + @@ -135609,12 +136477,12 @@ - - + + - - + + @@ -135661,24 +136529,24 @@ - - + + - - + + - - + + - - + + @@ -135698,13 +136566,13 @@ - - + + - - - + + + @@ -135718,16 +136586,16 @@ - - - - + + + + - - - - + + + + @@ -135735,6 +136603,21 @@ + + + + + + + + + + + + + + + @@ -135748,8 +136631,8 @@ - - + + @@ -135764,8 +136647,8 @@ - - + + @@ -135799,8 +136682,8 @@ - - + + @@ -135818,10 +136701,10 @@ - - - - + + + + @@ -135834,54 +136717,54 @@ - - + + - - - + + + - - + + - + - + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + + + + + + - - + + - - + + @@ -135890,36 +136773,36 @@ - - - - - + + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + @@ -135930,12 +136813,12 @@ - - - - - - + + + + + + @@ -135954,9 +136837,9 @@ - - - + + + @@ -135966,8 +136849,8 @@ - - + + @@ -135981,10 +136864,10 @@ - - - - + + + + @@ -135997,13 +136880,13 @@ - - - + + + - - + + @@ -136056,11 +136939,11 @@ - - - - - + + + + + @@ -136076,24 +136959,24 @@ - - + + - - + + - - + + - - + + @@ -136101,24 +136984,24 @@ - - + + - - + + - - + + @@ -136178,8 +137061,8 @@ - - + + @@ -136189,28 +137072,28 @@ - - - - - + + + + + - - - - - + + + + + - - - - + + + + @@ -136224,23 +137107,23 @@ - - - + + + - - + + - - - - + + + + - - + + @@ -136263,48 +137146,48 @@ - - - + + + - - - + + + - - - + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + @@ -136313,7 +137196,7 @@ - + @@ -136353,17 +137236,17 @@ - + - + - - + + @@ -136382,8 +137265,8 @@ - - + + @@ -136420,8 +137303,8 @@ - - + + @@ -136433,23 +137316,23 @@ - - - - + + + + - - - + + + - - - + + + @@ -136509,8 +137392,8 @@ - - + + @@ -136551,16 +137434,16 @@ - - + + - - + + - - + + @@ -136571,8 +137454,8 @@ - - + + @@ -136595,12 +137478,12 @@ - - + + - - + + @@ -136616,8 +137499,8 @@ - - + + @@ -136736,10 +137619,10 @@ - - - - + + + + @@ -136748,20 +137631,20 @@ - - - - - - - - + + + + + + + + - - - - + + + + @@ -136769,14 +137652,14 @@ - - - + + + - - - + + + @@ -136794,8 +137677,8 @@ - - + + @@ -136807,8 +137690,8 @@ - - + + @@ -136845,8 +137728,8 @@ - - + + @@ -136897,22 +137780,22 @@ - - - - + + + + - - - - + + + + - - - - + + + + @@ -137002,8 +137885,8 @@ - - + + @@ -137011,22 +137894,22 @@ - - + + - - - - - - + + + + + + - - - - + + + + @@ -137039,9 +137922,9 @@ - - - + + + @@ -137049,16 +137932,16 @@ - - - + + + - - - - - + + + + + @@ -137099,13 +137982,13 @@ - - - - - - - + + + + + + + @@ -137113,28 +137996,28 @@ - - + + - - + + - - + + - - + + - - + + @@ -137150,20 +138033,20 @@ - - - - - - + + + + + + - - - + + + - - + + @@ -137179,8 +138062,8 @@ - - + + @@ -137197,8 +138080,8 @@ - - + + @@ -137211,15 +138094,15 @@ - - - + + + - - - - + + + + @@ -137227,39 +138110,39 @@ - - - + + + - - - - + + + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + @@ -137267,30 +138150,30 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - + + - - + + @@ -137373,30 +138256,30 @@ - - + + - - - + + + - - - + + + - - - - - - + + + + + + - - + + @@ -137417,8 +138300,8 @@ - - + + @@ -137431,15 +138314,15 @@ - - - - - + + + + + - - + + @@ -137467,57 +138350,57 @@ - - + + - - + + - - + + - - - - - - - - - + + + + + + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - + + + - - + + @@ -137542,12 +138425,12 @@ - - + + - - + + @@ -137564,9 +138447,9 @@ - - - + + + @@ -137574,9 +138457,9 @@ - - - + + + @@ -137584,9 +138467,9 @@ - - - + + + @@ -137611,12 +138494,12 @@ - - + + - - + + @@ -137634,10 +138517,10 @@ - - - - + + + + @@ -137659,67 +138542,67 @@ - - - + + + - - - - + + + + - - - + + + - - + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + - - - + + + - - - + + + - - - + + + - - - - + + + + @@ -137730,13 +138613,13 @@ - + - - - - + + + + @@ -137770,10 +138653,10 @@ - - - - + + + + @@ -137815,7 +138698,7 @@ - + @@ -137823,34 +138706,34 @@ - - + + - - + + - - + + - - - - - - + + + + + + - - - - + + + + - - + + @@ -137861,10 +138744,10 @@ - - - - + + + + @@ -137874,8 +138757,8 @@ - - + + @@ -137886,8 +138769,8 @@ - - + + @@ -137925,8 +138808,8 @@ - - + + @@ -137964,14 +138847,14 @@ - - - - - + + + + + - - + + @@ -138009,11 +138892,11 @@ - - - - - + + + + + @@ -138157,21 +139040,21 @@ - - + + - - - + + + - - + + - - + + @@ -138205,11 +139088,11 @@ - - - - - + + + + + @@ -138224,21 +139107,21 @@ - - + + - - + + - - + + - - - + + + @@ -138254,11 +139137,11 @@ - - - - - + + + + + @@ -138282,12 +139165,12 @@ - - - - - - + + + + + + @@ -138307,24 +139190,30 @@ - - - - + + + + - - - - + + + + - - - + + + + + + + + + @@ -138342,26 +139231,26 @@ - - - - + + + + - - - + + + - - - - - + + + + + - - - + + + @@ -138370,20 +139259,20 @@ - - - + + + - - - + + + - - - + + + @@ -138393,26 +139282,26 @@ - - - + + + - - - + + + - - - - + + + + - - - - + + + + @@ -138558,20 +139447,26 @@ - - - + + + + + + + + + - - - + + + @@ -138579,26 +139474,26 @@ - - + + - - - - + + + + - - + + - - + + @@ -138606,20 +139501,25 @@ - - - - - - - - + + + - - - - + + + + + + + + + + + + + + @@ -138631,33 +139531,33 @@ - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + - - - + + + @@ -138665,28 +139565,28 @@ - - - + + + - - - - - + + + + + - - + + - - - + + + - - + + @@ -138697,13 +139597,13 @@ - - - - - - - + + + + + + + @@ -138735,14 +139635,14 @@ - - + + - - - - + + + + @@ -138750,10 +139650,10 @@ - - - - + + + + @@ -138761,10 +139661,10 @@ - - - - + + + + @@ -138787,10 +139687,10 @@ - - - - + + + + @@ -138806,6 +139706,10 @@ + + + + @@ -138813,29 +139717,29 @@ - + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -138864,13 +139768,13 @@ - - - - - - - + + + + + + + @@ -138889,10 +139793,10 @@ - - - - + + + + @@ -138909,10 +139813,10 @@ - - - - + + + + @@ -138922,30 +139826,30 @@ - - - + + + - - - - - - + + + + + + - - + + - - - - - - - + + + + + + + @@ -138962,13 +139866,13 @@ - - + + - - - + + + @@ -138987,39 +139891,39 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + - - + + @@ -139042,55 +139946,55 @@ - - - - - + + + + + - - - - - + + + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - + + - - - + + + - - - + + + @@ -139100,9 +140004,9 @@ - - - + + + @@ -139112,16 +140016,16 @@ - - - - - + + + + + - - - + + + @@ -139156,15 +140060,15 @@ - - - + + + - - - - + + + + @@ -139177,11 +140081,11 @@ - - - - - + + + + + @@ -139191,16 +140095,16 @@ - - - - - + + + + + - - - + + + @@ -139210,8 +140114,8 @@ - - + + @@ -139223,25 +140127,25 @@ - - + + - - + + - - - - - - - + + + + + + + - - + + @@ -139252,9 +140156,9 @@ - - - + + + @@ -139306,9 +140210,9 @@ - - - + + + @@ -139342,22 +140246,22 @@ - - - - - + + + + + - - - - - + + + + + - - + + @@ -139436,8 +140340,8 @@ - - + + @@ -139447,9 +140351,9 @@ - - - + + + @@ -139501,8 +140405,8 @@ - - + + @@ -139518,9 +140422,9 @@ - - - + + + @@ -139530,17 +140434,17 @@ - - - - + + + + - - - - - + + + + + @@ -139564,17 +140468,17 @@ - - + + - - - - - - - + + + + + + + @@ -139583,23 +140487,23 @@ - - + + - - - - + + + + - - - + + + - - + + @@ -139610,12 +140514,12 @@ - - - - - - + + + + + + @@ -139627,8 +140531,8 @@ - - + + @@ -139640,18 +140544,18 @@ - - - - - - + + + + + + - - - - + + + + @@ -139659,12 +140563,12 @@ - - + + - - + + @@ -139702,9 +140606,9 @@ - - - + + + @@ -139716,8 +140620,8 @@ - - + + @@ -139749,45 +140653,45 @@ - - - + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + + - - - - - + + + + + @@ -139832,9 +140736,9 @@ - - - + + + @@ -139865,10 +140769,10 @@ - - - - + + + + @@ -139960,14 +140864,14 @@ - - - - + + + + - - + + @@ -139985,8 +140889,8 @@ - - + + @@ -140237,9 +141141,9 @@ - - - + + + @@ -140328,9 +141232,9 @@ - - - + + + @@ -140351,13 +141255,13 @@ - - - - - - - + + + + + + + @@ -140369,44 +141273,44 @@ - - - + + + - - - - + + + + - - + + - - + + - - - - - - + + + + + + - - + + - - - + + + @@ -140516,9 +141420,9 @@ - - - + + + @@ -140530,17 +141434,17 @@ - - - - - - + + + + + + - - - + + + @@ -140664,9 +141568,9 @@ - - - + + + @@ -140699,34 +141603,34 @@ - - + + - - + + - - + + - - + + - - - - + + + + - - - - - - + + + + + + @@ -140749,9 +141653,9 @@ - - - + + + @@ -140769,8 +141673,8 @@ - - + + @@ -140805,10 +141709,10 @@ - - - - + + + + @@ -140817,21 +141721,21 @@ - - - - + + + + - - - + + + - - - - + + + + @@ -140839,21 +141743,21 @@ - - - - + + + + - - - - - + + + + + - - - + + + @@ -140872,16 +141776,16 @@ - - - + + + - - - + + + - - + + @@ -140899,43 +141803,47 @@ - - + + - - + + - - + + - - + + - - + + - - - + + + + + + + - + - + @@ -140947,12 +141855,12 @@ - - + + - - + + @@ -140967,23 +141875,23 @@ - - + + - - + + - - - + + + - + - + @@ -141002,15 +141910,15 @@ - - - + + + - - - - + + + + @@ -141023,27 +141931,27 @@ - - - + + + - - - - + + + + - - - + + + - - - - + + + + @@ -141065,9 +141973,9 @@ - - - + + + @@ -141076,10 +141984,10 @@ - - - - + + + + @@ -141087,9 +141995,9 @@ - - - + + + @@ -141097,9 +142005,9 @@ - - - + + + @@ -141108,17 +142016,17 @@ - - - - + + + + - + @@ -141136,8 +142044,8 @@ - - + + @@ -141171,6 +142079,7 @@ + @@ -141179,8 +142088,8 @@ - - + + @@ -141273,8 +142182,16 @@ - - + + + + + + + + + + @@ -141283,8 +142200,8 @@ - - + + @@ -141316,19 +142233,19 @@ - - - - - - - - - + + + + + + + + + - - + + @@ -141340,9 +142257,9 @@ - - - + + + @@ -141361,14 +142278,14 @@ - - - - + + + + - - + + @@ -141396,9 +142313,9 @@ - - - + + + @@ -141421,12 +142338,11 @@ - - - + + @@ -141461,19 +142377,19 @@ - - + + - - - - + + + + - - - + + + @@ -141491,8 +142407,8 @@ - - + + @@ -141502,24 +142418,24 @@ - - - - - + + + + + - - - - + + + + - - - - - + + + + + @@ -141529,26 +142445,26 @@ - - - - + + + + - - - + + + - - - + + + - - - + + + @@ -141557,10 +142473,10 @@ - - - - + + + + @@ -141577,7 +142493,7 @@ - + @@ -141611,9 +142527,9 @@ - - - + + + @@ -141751,10 +142667,10 @@ - - - - + + + + @@ -141762,10 +142678,10 @@ - - - - + + + + @@ -141844,19 +142760,19 @@ - - - + + + - - - + + + - - - + + + @@ -141886,8 +142802,8 @@ - - + + @@ -141905,16 +142821,16 @@ - - + + - - + + @@ -141967,9 +142883,9 @@ - - - + + + @@ -141983,9 +142899,9 @@ - - - + + + @@ -142024,8 +142940,8 @@ - - + + @@ -142071,9 +142987,9 @@ - - - + + + @@ -142164,11 +143080,11 @@ - - - - - + + + + + @@ -142403,9 +143319,9 @@ - - - + + + @@ -142523,27 +143439,27 @@ - - + + - - + + - - + + - - - - - + + + + + @@ -142577,8 +143493,8 @@ - - + + @@ -142617,17 +143533,17 @@ - - + + - - + + - - - + + + @@ -142638,8 +143554,8 @@ - - + + @@ -142670,12 +143586,12 @@ - - + + - - + + @@ -142698,8 +143614,8 @@ - - + + @@ -142721,8 +143637,8 @@ - - + + @@ -142758,9 +143674,9 @@ - - - + + + @@ -142783,9 +143699,9 @@ - - - + + + @@ -142800,16 +143716,16 @@ - - + + - - + + - - + + @@ -142828,8 +143744,8 @@ - - + + @@ -142844,16 +143760,16 @@ - - - - - - - - - - + + + + + + + + + + @@ -142887,8 +143803,8 @@ - - + + @@ -142903,15 +143819,15 @@ - - - - + + + + - - - + + + @@ -142922,9 +143838,9 @@ - - - + + + @@ -142936,9 +143852,9 @@ - - - + + + @@ -142957,24 +143873,24 @@ - - - + + + - - - - + + + + - - - + + + @@ -143001,8 +143917,8 @@ - - + + @@ -143021,9 +143937,9 @@ - - - + + + @@ -143032,28 +143948,28 @@ - - - - + + + + - - + + - - + + - - - + + + - - - + + + @@ -143062,14 +143978,14 @@ - - - - + + + + - - + + @@ -143080,9 +143996,9 @@ - - - + + + @@ -143097,8 +144013,8 @@ - - + + @@ -143120,8 +144036,8 @@ - - + + @@ -143224,15 +144140,15 @@ - - - + + + - - - - + + + + @@ -143244,8 +144160,8 @@ - - + + @@ -143311,16 +144227,16 @@ - - + + - - + + @@ -143328,10 +144244,10 @@ - - - - + + + + @@ -143355,38 +144271,38 @@ - - + + - - + + - - - + + + - - - + + + - - + + - - + + @@ -143394,10 +144310,10 @@ - - - - + + + + @@ -143421,9 +144337,9 @@ - - - + + + @@ -143431,8 +144347,8 @@ - - + + @@ -143481,9 +144397,9 @@ - - - + + + @@ -143504,10 +144420,10 @@ - - - - + + + + @@ -143557,9 +144473,9 @@ - - - + + + @@ -143619,50 +144535,50 @@ - - - + + + - - - - - - + + + + + + - - - - - + + + + + - - + + - - - - + + + + - - - - - - + + + + + + - - + + @@ -143838,9 +144754,9 @@ - - - + + + @@ -143854,27 +144770,27 @@ - - + + - - - - + + + + - - - + + + @@ -143888,33 +144804,33 @@ - - - - - + + + + + - - - + + + - - + + - - + + - - + + - - - + + + @@ -143945,15 +144861,15 @@ - - - - + + + + - - - + + + @@ -143965,30 +144881,30 @@ - - - - + + + + - - + + - - - - + + + + - - - + + + - - - + + + @@ -144000,62 +144916,62 @@ - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -144063,42 +144979,42 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + @@ -144107,24 +145023,24 @@ - - + + - - - - + + + + - - - - + + + + @@ -144135,8 +145051,8 @@ - - + + @@ -144160,15 +145076,15 @@ - - - + + + - - - - + + + + @@ -144219,15 +145135,15 @@ - - - + + + - - - + + + @@ -144240,15 +145156,15 @@ - - - - + + + + - - - + + + @@ -144326,9 +145242,9 @@ - - - + + + @@ -144344,36 +145260,36 @@ - - - + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + @@ -144388,9 +145304,9 @@ - - - + + + @@ -144405,14 +145321,14 @@ - - - + + + - - - + + + @@ -144454,15 +145370,15 @@ - - + + - - - - - + + + + + @@ -144470,11 +145386,11 @@ - - - - - + + + + + @@ -144484,14 +145400,32 @@ + + + + + + + + + + + + - - + + + + + + + + @@ -144511,8 +145445,8 @@ - - + + @@ -144555,8 +145489,8 @@ - - + + @@ -144592,8 +145526,8 @@ - - + + @@ -144660,9 +145594,9 @@ - - - + + + @@ -144733,26 +145667,26 @@ - - - - - - - + + + + + + + - - + + - - - + + + @@ -144789,21 +145723,21 @@ - - - + + + - - - + + + - - - + + + @@ -144870,15 +145804,15 @@ - - + + - - + + @@ -144889,16 +145823,16 @@ - - - - - + + + + + - - - + + + @@ -144907,11 +145841,11 @@ - - - - - + + + + + @@ -144920,9 +145854,9 @@ - - - + + + @@ -144972,11 +145906,11 @@ - - - - - + + + + + @@ -144986,23 +145920,23 @@ - - - - + + + + - - - + + + - - + + - - + + @@ -145021,12 +145955,12 @@ - - + + - - + + @@ -145034,8 +145968,8 @@ - - + + @@ -145056,8 +145990,8 @@ - - + + @@ -145065,16 +145999,16 @@ - - - + + + - + - - + + @@ -145082,8 +146016,8 @@ - - + + @@ -145099,9 +146033,9 @@ - - - + + + @@ -145124,17 +146058,17 @@ - - - + + + - - + + @@ -145151,12 +146085,12 @@ - - - - - - + + + + + + @@ -145178,6 +146112,12 @@ + + + + + + @@ -145211,9 +146151,9 @@ - - - + + + @@ -145238,34 +146178,34 @@ - - - + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + + - - - + + + @@ -145275,17 +146215,17 @@ - - + + - - - - + + + + @@ -145296,26 +146236,26 @@ - - - - + + + + - - - - + + + + - - + + - - - - + + + + @@ -145326,9 +146266,9 @@ - - - + + + @@ -145360,10 +146300,10 @@ - - - - + + + + @@ -145396,9 +146336,9 @@ - - - + + + @@ -145415,15 +146355,15 @@ - - - - + + + + - - - + + + @@ -145447,12 +146387,12 @@ - - - - - - + + + + + + @@ -145474,13 +146414,13 @@ - - + + - - - + + + @@ -145497,10 +146437,10 @@ - - - - + + + + @@ -145511,47 +146451,47 @@ - - - + + + - - + + - - - - - - - + + + + + + + - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + diff --git a/android/abi_gki_aarch64_galaxy b/android/abi_gki_aarch64_galaxy index f35385449132..d5358ae912e2 100644 --- a/android/abi_gki_aarch64_galaxy +++ b/android/abi_gki_aarch64_galaxy @@ -2027,10 +2027,12 @@ gpiochip_generic_free gpiochip_generic_request gpiochip_get_data + gpiochip_irqchip_add_key gpiochip_line_is_valid gpiochip_lock_as_irq gpiochip_populate_parent_fwspec_fourcell gpiochip_remove + gpiochip_set_nested_irqchip gpiochip_unlock_as_irq gpiod_cansleep gpiod_count @@ -2053,9 +2055,14 @@ gro_cells_destroy gro_cells_init gro_cells_receive + gs_alloc_req + gs_free_req gserial_alloc_line gserial_connect gserial_disconnect + gserial_free_line + gserial_resume + gserial_suspend guid_gen handle_bad_irq handle_edge_irq diff --git a/android/abi_gki_aarch64_oplus b/android/abi_gki_aarch64_oplus index bf898882c7b3..72015e048a60 100644 --- a/android/abi_gki_aarch64_oplus +++ b/android/abi_gki_aarch64_oplus @@ -63,6 +63,7 @@ bdput bio_add_pc_page bio_alloc_bioset + bio_associate_blkg_from_css bio_endio bio_put bio_reset @@ -83,6 +84,7 @@ bitmap_zalloc blk_alloc_queue blk_cleanup_queue + blkdev_issue_zeroout blk_execute_rq blk_execute_rq_nowait blk_get_request @@ -90,6 +92,7 @@ blk_mq_rq_cpu blk_mq_sched_mark_restart_hctx blk_mq_start_request + blk_op_str blk_put_request blk_queue_flag_clear blk_queue_flag_set @@ -152,6 +155,7 @@ class_unregister __cleancache_get_page cleanup_srcu_struct + clear_nlink clear_page __ClearPageMovable clk_bulk_disable @@ -372,6 +376,7 @@ del_gendisk del_timer del_timer_sync + dentry_path_raw desc_to_gpio destroy_workqueue dev_alloc_name @@ -590,6 +595,9 @@ devres_release dev_set_name _dev_warn + d_find_alias + d_instantiate_new + d_invalidate disable_irq disable_irq_nosync disable_percpu_irq @@ -681,6 +689,39 @@ down_write d_path dput + dqget + dqput + dquot_acquire + dquot_alloc + dquot_alloc_inode + __dquot_alloc_space + dquot_claim_space_nodirty + dquot_commit + dquot_commit_info + dquot_destroy + dquot_disable + dquot_drop + dquot_file_open + dquot_free_inode + __dquot_free_space + dquot_get_dqblk + dquot_get_next_dqblk + dquot_get_next_id + dquot_get_state + dquot_initialize + dquot_initialize_needed + dquot_load_quota_inode + dquot_mark_dquot_dirty + dquot_quota_off + dquot_quota_on + dquot_quota_on_mount + dquot_release + dquot_resume + dquot_set_dqblk + dquot_set_dqinfo + __dquot_transfer + dquot_transfer + dquot_writeback_dquots drain_workqueue driver_create_file driver_find_device @@ -820,6 +861,7 @@ drm_writeback_connector_init drm_writeback_queue_job drm_writeback_signal_completion + d_tmpfile dump_stack __dynamic_pr_debug edac_device_add_device @@ -850,6 +892,7 @@ eventfd_ctx_remove_wait_queue eventfd_signal event_triggers_call + evict_inodes extcon_get_edev_by_phandle extcon_get_edev_name extcon_get_property @@ -867,6 +910,7 @@ filp_close filp_open_block find_get_pid + find_inode_nowait find_last_bit find_next_bit find_next_zero_bit @@ -897,6 +941,8 @@ free_pages_exact free_percpu free_percpu_irq + freeze_bdev + freeze_super freezing_slow_path freq_qos_add_notifier freq_qos_add_request @@ -905,11 +951,60 @@ freq_qos_update_request freq_scale fs_bio_set + fscrypt_decrypt_bio + fscrypt_dio_supported + fscrypt_drop_inode + fscrypt_encrypt_pagecache_blocks + __fscrypt_encrypt_symlink + fscrypt_file_open + fscrypt_fname_alloc_buffer + fscrypt_fname_disk_to_usr + fscrypt_fname_free_buffer + fscrypt_fname_siphash + fscrypt_free_bounce_page + fscrypt_free_inode + fscrypt_get_symlink + fscrypt_has_permitted_context + __fscrypt_inode_uses_inline_crypto + fscrypt_ioctl_add_key + fscrypt_ioctl_get_key_status + fscrypt_ioctl_get_nonce + fscrypt_ioctl_get_policy + fscrypt_ioctl_get_policy_ex + fscrypt_ioctl_remove_key + fscrypt_ioctl_remove_key_all_users + fscrypt_ioctl_set_policy + fscrypt_match_name + fscrypt_mergeable_bio + __fscrypt_prepare_link + __fscrypt_prepare_lookup + fscrypt_prepare_new_inode + __fscrypt_prepare_readdir + __fscrypt_prepare_rename + __fscrypt_prepare_setattr + fscrypt_prepare_symlink + fscrypt_put_encryption_info + fscrypt_set_bio_crypt_ctx + fscrypt_set_context + fscrypt_set_test_dummy_encryption + fscrypt_setup_filename + fscrypt_show_test_dummy_encryption + fscrypt_symlink_getattr + fscrypt_zeroout_range fsg_common_create_luns fsg_common_set_cdev fsg_common_set_inquiry_string fsg_common_set_sysfs fsg_config_from_params + fsverity_cleanup_inode + fsverity_enqueue_verify_work + fsverity_file_open + fsverity_ioctl_enable + fsverity_ioctl_measure + fsverity_ioctl_read_metadata + fsverity_prepare_setattr + fsverity_verify_bio + fsverity_verify_page fsync_bdev fwnode_device_is_available fwnode_find_reference @@ -930,10 +1025,15 @@ gcd generate_random_uuid generic_device_group + generic_fh_to_dentry + generic_fh_to_parent generic_file_llseek + generic_file_llseek_size + __generic_file_write_iter generic_handle_irq generic_iommu_put_resv_regions generic_mii_ioctl + generic_set_encrypted_ci_d_ops genlmsg_multicast_allns genlmsg_put genl_register_family @@ -1089,6 +1189,8 @@ idr_preload idr_remove idr_replace + iget_failed + iget_locked iio_alloc_pollfunc iio_buffer_init iio_buffer_put @@ -1106,11 +1208,14 @@ iio_read_channel_processed iio_read_channel_raw iio_trigger_notify_done + ilookup import_iovec inc_node_page_state inc_zone_page_state in_egroup_p + __inet6_lookup_established inet_csk_get_port + __inet_lookup_established init_iova_domain init_net init_pseudo @@ -1122,6 +1227,7 @@ init_uts_ns init_wait_entry __init_waitqueue_head + inode_nohighmem inode_owner_or_capable inode_permission input_alloc_absinfo @@ -1143,6 +1249,7 @@ input_unregister_device input_unregister_handle input_unregister_handler + insert_inode_locked interval_tree_insert interval_tree_iter_first interval_tree_iter_next @@ -1874,6 +1981,9 @@ pm_wakeup_dev_event pm_wakeup_ws_event pm_wq + posix_acl_alloc + posix_acl_chmod + posix_acl_equiv_mode power_supply_changed power_supply_get_by_name power_supply_get_drvdata @@ -2235,6 +2345,7 @@ send_sig send_sig_info seq_buf_printf + seq_escape seq_hex_dump seq_lseek seq_open @@ -2257,11 +2368,13 @@ serial8250_rpm_put serial8250_suspend_port serial8250_unregister_port + set_cached_acl set_cpus_allowed_ptr set_normalized_timespec64 set_page_dirty_lock __SetPageMovable set_task_cpu + set_task_ioprio set_user_nice sg_alloc_table sg_alloc_table_from_pages @@ -2277,6 +2390,7 @@ __sg_page_iter_start sg_scsi_ioctl show_regs + shrink_dcache_sb shrink_slab sigprocmask si_mem_available @@ -2469,6 +2583,7 @@ synchronize_net synchronize_rcu synchronize_srcu + sync_inodes_sb syscon_node_to_regmap syscon_regmap_lookup_by_compatible syscon_regmap_lookup_by_phandle @@ -2514,7 +2629,10 @@ __task_pid_nr_ns __task_rq_lock task_rq_lock + tcp_hashinfo tcp_parse_options + thaw_bdev + thaw_super thermal_cooling_device_register thermal_cooling_device_unregister thermal_of_cooling_device_register @@ -2630,10 +2748,10 @@ __traceiter_android_vh_binder_reply __traceiter_android_vh_binder_restore_priority __traceiter_android_vh_binder_set_priority - __traceiter_android_vh_binder_trans - __traceiter_android_vh_binder_transaction_init __traceiter_android_vh_binder_thread_read __traceiter_android_vh_binder_thread_release + __traceiter_android_vh_binder_trans + __traceiter_android_vh_binder_transaction_init __traceiter_android_vh_binder_wait_for_work __traceiter_android_vh_binder_wakeup_ilocked __traceiter_android_vh_build_sched_domains @@ -2645,6 +2763,7 @@ __traceiter_android_vh_check_uninterruptible_tasks_dn __traceiter_android_vh_clear_mask_adjust __traceiter_android_vh_clear_reserved_fmt_fields + __traceiter_android_vh_cma_drain_all_pages_bypass __traceiter_android_vh_commit_creds __traceiter_android_vh_cpufreq_acct_update_power __traceiter_android_vh_cpufreq_fast_switch @@ -2654,6 +2773,7 @@ __traceiter_android_vh_cpu_idle_exit __traceiter_android_vh_cpu_up __traceiter_android_vh_do_send_sig_info + __traceiter_android_vh_drain_all_pages_bypass __traceiter_android_vh_em_cpu_energy __traceiter_android_vh_exclude_reserved_zone __traceiter_android_vh_exit_creds @@ -2687,6 +2807,8 @@ __traceiter_android_vh_mutex_wait_finish __traceiter_android_vh_mutex_wait_start __traceiter_android_vh_override_creds + __traceiter_android_vh_page_referenced_check_bypass + __traceiter_android_vh_pcplist_add_cma_pages_bypass __traceiter_android_vh_prepare_update_load_avg_se __traceiter_android_vh_printk_hotplug __traceiter_android_vh_process_killed @@ -2724,10 +2846,6 @@ __traceiter_android_vh_tune_inactive_ratio __traceiter_android_vh_tune_scan_type __traceiter_android_vh_tune_swappiness - __traceiter_android_vh_page_referenced_check_bypass - __traceiter_android_vh_drain_all_pages_bypass - __traceiter_android_vh_cma_drain_all_pages_bypass - __traceiter_android_vh_pcplist_add_cma_pages_bypass __traceiter_android_vh_ufs_compl_command __traceiter_android_vh_ufs_send_command __traceiter_android_vh_ufs_send_tm_command @@ -2852,6 +2970,7 @@ __tracepoint_android_vh_check_uninterruptible_tasks_dn __tracepoint_android_vh_clear_mask_adjust __tracepoint_android_vh_clear_reserved_fmt_fields + __tracepoint_android_vh_cma_drain_all_pages_bypass __tracepoint_android_vh_commit_creds __tracepoint_android_vh_cpufreq_acct_update_power __tracepoint_android_vh_cpufreq_fast_switch @@ -2861,6 +2980,7 @@ __tracepoint_android_vh_cpu_idle_exit __tracepoint_android_vh_cpu_up __tracepoint_android_vh_do_send_sig_info + __tracepoint_android_vh_drain_all_pages_bypass __tracepoint_android_vh_em_cpu_energy __tracepoint_android_vh_exclude_reserved_zone __tracepoint_android_vh_exit_creds @@ -2894,6 +3014,8 @@ __tracepoint_android_vh_mutex_wait_finish __tracepoint_android_vh_mutex_wait_start __tracepoint_android_vh_override_creds + __tracepoint_android_vh_page_referenced_check_bypass + __tracepoint_android_vh_pcplist_add_cma_pages_bypass __tracepoint_android_vh_prepare_update_load_avg_se __tracepoint_android_vh_printk_hotplug __tracepoint_android_vh_process_killed @@ -2931,10 +3053,6 @@ __tracepoint_android_vh_tune_inactive_ratio __tracepoint_android_vh_tune_scan_type __tracepoint_android_vh_tune_swappiness - __tracepoint_android_vh_page_referenced_check_bypass - __tracepoint_android_vh_drain_all_pages_bypass - __tracepoint_android_vh_cma_drain_all_pages_bypass - __tracepoint_android_vh_pcplist_add_cma_pages_bypass __tracepoint_android_vh_ufs_compl_command __tracepoint_android_vh_ufs_send_command __tracepoint_android_vh_ufs_send_tm_command @@ -3325,6 +3443,9 @@ verify_pkcs7_signature vfree vfs_fsync + vfs_ioc_fssetxattr_check + vfs_ioc_setflags_prepare + vfs_setpos video_devdata video_device_alloc video_device_release @@ -3391,6 +3512,7 @@ __warn_printk watchdog_init_timeout watchdog_set_restart_priority + wbc_account_cgroup_owner wireless_nlevent_flush woken_wake_function work_busy diff --git a/android/abi_gki_aarch64_qcom b/android/abi_gki_aarch64_qcom index bcfc4c9ef782..fe1688823b24 100644 --- a/android/abi_gki_aarch64_qcom +++ b/android/abi_gki_aarch64_qcom @@ -2603,12 +2603,14 @@ __traceiter_ipi_entry __traceiter_ipi_raise __traceiter_irq_handler_entry + __traceiter_map __traceiter_rwmmio_post_read __traceiter_rwmmio_read __traceiter_rwmmio_write __traceiter_sched_overutilized_tp __traceiter_sched_switch __traceiter_suspend_resume + __traceiter_unmap __tracepoint_android_rvh_account_irq __tracepoint_android_rvh_after_dequeue_task __tracepoint_android_rvh_after_enqueue_task @@ -2727,6 +2729,7 @@ __tracepoint_ipi_entry __tracepoint_ipi_raise __tracepoint_irq_handler_entry + __tracepoint_map tracepoint_probe_register tracepoint_probe_register_prio tracepoint_probe_unregister @@ -2736,6 +2739,7 @@ __tracepoint_sched_overutilized_tp __tracepoint_sched_switch __tracepoint_suspend_resume + __tracepoint_unmap trace_print_array_seq trace_print_flags_seq trace_print_hex_seq diff --git a/arch/arm64/configs/gki_defconfig b/arch/arm64/configs/gki_defconfig index 36901bc73efd..7d7c4086de6c 100644 --- a/arch/arm64/configs/gki_defconfig +++ b/arch/arm64/configs/gki_defconfig @@ -118,6 +118,9 @@ CONFIG_CMA_DEBUGFS=y CONFIG_CMA_SYSFS=y CONFIG_CMA_AREAS=16 CONFIG_READ_ONLY_THP_FOR_FS=y +CONFIG_DAMON=y +CONFIG_DAMON_PADDR=y +CONFIG_DAMON_RECLAIM=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -652,6 +655,7 @@ CONFIG_CRYPTO_ADIANTUM=y CONFIG_CRYPTO_XCBC=y CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_DES=y CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y diff --git a/arch/x86/configs/gki_defconfig b/arch/x86/configs/gki_defconfig index c9ce1777eb58..14ea69c0d417 100644 --- a/arch/x86/configs/gki_defconfig +++ b/arch/x86/configs/gki_defconfig @@ -94,6 +94,9 @@ CONFIG_CMA_DEBUGFS=y CONFIG_CMA_SYSFS=y CONFIG_CMA_AREAS=16 CONFIG_READ_ONLY_THP_FOR_FS=y +CONFIG_DAMON=y +CONFIG_DAMON_PADDR=y +CONFIG_DAMON_RECLAIM=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -586,6 +589,7 @@ CONFIG_CRYPTO_BLAKE2B=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_SHA256_SSSE3=y CONFIG_CRYPTO_AES_NI_INTEL=y +CONFIG_CRYPTO_DES=y CONFIG_CRYPTO_LZ4=y CONFIG_CRYPTO_ZSTD=y CONFIG_CRYPTO_ANSI_CPRNG=y diff --git a/arch/x86/include/asm/kfence.h b/arch/x86/include/asm/kfence.h index 97bbb4a9083a..05b48b33baf0 100644 --- a/arch/x86/include/asm/kfence.h +++ b/arch/x86/include/asm/kfence.h @@ -56,8 +56,13 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) else set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT)); - /* Flush this CPU's TLB. */ + /* + * Flush this CPU's TLB, assuming whoever did the allocation/free is + * likely to continue running on this CPU. + */ + preempt_disable(); flush_tlb_one_kernel(addr); + preempt_enable(); return true; } diff --git a/build.config.common b/build.config.common index b321ae20ecea..9ec1c637a79f 100644 --- a/build.config.common +++ b/build.config.common @@ -7,6 +7,7 @@ DTC=dtc CLANG_PREBUILT_BIN=prebuilts-master/clang/host/linux-x86/clang-r416183b/bin BUILDTOOLS_PREBUILT_BIN=build/build-tools/path/linux-x86 +KCFLAGS="${KCFLAGS} -D__ANDROID_COMMON_KERNEL__" EXTRA_CMDS='' STOP_SHIP_TRACEPRINTK=1 IN_KERNEL_MODULES=1 diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig index 38485f83c7d4..b5aa168f27f6 100644 --- a/drivers/android/Kconfig +++ b/drivers/android/Kconfig @@ -74,14 +74,32 @@ config ANDROID_VENDOR_HOOKS Allow vendor modules to attach to tracepoint "hooks" defined via DECLARE_HOOK or DECLARE_RESTRICTED_HOOK. -config ANDROID_STRUCT_PADDING - bool "Android Struct Padding" +config ANDROID_KABI_RESERVE + bool "Android KABI reserve padding" + default y if !NO_GKI + help + This option enables the padding that the Android GKI kernel adds + to many different kernel structures to support an in-kernel stable ABI + over the lifespan of support for the kernel. + + Only disable this option if you have a system that needs the Android + kernel drivers, but is NOT an Android GKI kernel image. If disabled + it has the possibility to make the kernel static and runtime image + slightly smaller but will NOT be supported by the Google Android + kernel team. + + If even slightly unsure, say Y. + +config ANDROID_VENDOR_OEM_DATA + bool "Android vendor and OEM data padding" default y if !NO_GKI help This option enables the padding that the Android GKI kernel adds to many different kernel structures to support an in-kernel stable ABI over the lifespan of support for the kernel as well as OEM additional - fields that are needed by some of the Android kernel tracepoints. + fields that are needed by some of the Android kernel tracepoints. The + macros enabled by this option are used to enable padding in vendor modules + used for the above specified purposes. Only disable this option if you have a system that needs the Android kernel drivers, but is NOT an Android GKI kernel image and you do NOT diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c index 2ae0b6eecc37..de3a032822f5 100644 --- a/drivers/android/vendor_hooks.c +++ b/drivers/android/vendor_hooks.c @@ -113,6 +113,11 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_read_wait_finish); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_start); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_write_wait_finish); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_set_owner); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_set_reader_owned); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_mark_wake_readers); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_up_read_end); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_up_write_end); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_sched_show_task); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shmem_alloc_page); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cpu_idle_enter); @@ -397,3 +402,5 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_has_work_ilocked); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_binder_read_done); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_handle_tlb_conf); EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_shrink_node_memcgs); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ra_tuning_max_page); +EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_tune_memcg_scan_type); diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 018fb172a098..132b019695f3 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -827,9 +827,17 @@ static int virtblk_probe(struct virtio_device *vdev) err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE, struct virtio_blk_config, blk_size, &blk_size); - if (!err) + if (!err) { + err = blk_validate_block_size(blk_size); + if (err) { + dev_err(&vdev->dev, + "virtio_blk: invalid block size: 0x%x\n", + blk_size); + goto out_cleanup_disk; + } + blk_queue_logical_block_size(q, blk_size); - else + } else blk_size = queue_logical_block_size(q); /* Use topology information if available */ @@ -890,6 +898,8 @@ static int virtblk_probe(struct virtio_device *vdev) device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups); return 0; +out_cleanup_disk: + blk_cleanup_queue(vblk->disk->queue); out_free_tags: blk_mq_free_tag_set(&vblk->tag_set); out_put_disk: diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 0853dd704d81..52eb5f1fd023 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -268,11 +268,13 @@ int iommu_probe_device(struct device *dev) * support default domains, so the return value is not yet * checked. */ + mutex_lock(&group->mutex); iommu_alloc_default_domain(group, dev); if (group->default_domain) { ret = __iommu_attach_device(group->default_domain, dev); if (ret) { + mutex_unlock(&group->mutex); iommu_group_put(group); goto err_release; } @@ -280,6 +282,7 @@ int iommu_probe_device(struct device *dev) iommu_create_device_direct_mappings(group, dev); + mutex_unlock(&group->mutex); iommu_group_put(group); if (ops->probe_finalize) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 10eb0544f1cb..9fd176e225af 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -608,18 +608,17 @@ static void start_io_acct(struct dm_io *io) false, 0, &io->stats_aux); } -static void end_io_acct(struct dm_io *io) +static void end_io_acct(struct mapped_device *md, struct bio *bio, + unsigned long start_time, struct dm_stats_aux *stats_aux) { - struct mapped_device *md = io->md; - struct bio *bio = io->orig_bio; - unsigned long duration = jiffies - io->start_time; + unsigned long duration = jiffies - start_time; - bio_end_io_acct(bio, io->start_time); + bio_end_io_acct(bio, start_time); if (unlikely(dm_stats_used(&md->stats))) dm_stats_account_io(&md->stats, bio_data_dir(bio), bio->bi_iter.bi_sector, bio_sectors(bio), - true, duration, &io->stats_aux); + true, duration, stats_aux); /* nudge anyone waiting on suspend queue */ if (unlikely(wq_has_sleeper(&md->wait))) @@ -904,6 +903,8 @@ static void dec_pending(struct dm_io *io, blk_status_t error) blk_status_t io_error; struct bio *bio; struct mapped_device *md = io->md; + unsigned long start_time = 0; + struct dm_stats_aux stats_aux; /* Push-back supersedes any I/O errors */ if (unlikely(error)) { @@ -930,8 +931,10 @@ static void dec_pending(struct dm_io *io, blk_status_t error) io_error = io->status; bio = io->orig_bio; - end_io_acct(io); + start_time = io->start_time; + stats_aux = io->stats_aux; free_io(md, io); + end_io_acct(md, bio, start_time, &stats_aux); if (io_error == BLK_STS_DM_REQUEUE) return; diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c index b221b4e438a1..73190652c267 100644 --- a/drivers/media/v4l2-core/v4l2-mem2mem.c +++ b/drivers/media/v4l2-core/v4l2-mem2mem.c @@ -585,19 +585,14 @@ int v4l2_m2m_reqbufs(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, } EXPORT_SYMBOL_GPL(v4l2_m2m_reqbufs); -int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, - struct v4l2_buffer *buf) +static void v4l2_m2m_adjust_mem_offset(struct vb2_queue *vq, + struct v4l2_buffer *buf) { - struct vb2_queue *vq; - int ret = 0; - unsigned int i; - - vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); - ret = vb2_querybuf(vq, buf); - /* Adjust MMAP memory offsets for the CAPTURE queue */ if (buf->memory == V4L2_MEMORY_MMAP && V4L2_TYPE_IS_CAPTURE(vq->type)) { if (V4L2_TYPE_IS_MULTIPLANAR(vq->type)) { + unsigned int i; + for (i = 0; i < buf->length; ++i) buf->m.planes[i].m.mem_offset += DST_QUEUE_OFF_BASE; @@ -605,8 +600,23 @@ int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, buf->m.offset += DST_QUEUE_OFF_BASE; } } +} - return ret; +int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, + struct v4l2_buffer *buf) +{ + struct vb2_queue *vq; + int ret; + + vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); + ret = vb2_querybuf(vq, buf); + if (ret) + return ret; + + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + + return 0; } EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf); @@ -763,6 +773,9 @@ int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, if (ret) return ret; + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + /* * If the capture queue is streaming, but streaming hasn't started * on the device, but was asked to stop, mark the previously queued @@ -784,9 +797,17 @@ int v4l2_m2m_dqbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, struct v4l2_buffer *buf) { struct vb2_queue *vq; + int ret; vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); - return vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK); + ret = vb2_dqbuf(vq, buf, file->f_flags & O_NONBLOCK); + if (ret) + return ret; + + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + + return 0; } EXPORT_SYMBOL_GPL(v4l2_m2m_dqbuf); @@ -795,9 +816,17 @@ int v4l2_m2m_prepare_buf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, { struct video_device *vdev = video_devdata(file); struct vb2_queue *vq; + int ret; vq = v4l2_m2m_get_vq(m2m_ctx, buf->type); - return vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf); + ret = vb2_prepare_buf(vq, vdev->v4l2_dev->mdev, buf); + if (ret) + return ret; + + /* Adjust MMAP memory offsets for the CAPTURE queue */ + v4l2_m2m_adjust_mem_offset(vq, buf); + + return 0; } EXPORT_SYMBOL_GPL(v4l2_m2m_prepare_buf); diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index f9612eeb714e..8dd8b9138789 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -59,6 +59,7 @@ static int rproc_release_carveout(struct rproc *rproc, /* Unique indices for remoteproc devices */ static DEFINE_IDA(rproc_dev_index); +static struct workqueue_struct *rproc_recovery_wq; static const char * const rproc_crash_names[] = { [RPROC_MMUFAULT] = "mmufault", @@ -2488,8 +2489,11 @@ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type) dev_err(&rproc->dev, "crash detected in %s: type %s\n", rproc->name, rproc_crash_to_string(type)); + if (rproc_recovery_wq) + queue_work(rproc_recovery_wq, &rproc->crash_handler); + else /* Have a worker handle the error; ensure system is not suspended */ - queue_work(system_freezable_wq, &rproc->crash_handler); + queue_work(system_freezable_wq, &rproc->crash_handler); } EXPORT_SYMBOL(rproc_report_crash); @@ -2534,6 +2538,11 @@ static void __exit rproc_exit_panic(void) static int __init remoteproc_init(void) { + rproc_recovery_wq = alloc_workqueue("rproc_recovery_wq", + WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!rproc_recovery_wq) + pr_err("remoteproc: creation of rproc_recovery_wq failed\n"); + rproc_init_sysfs(); rproc_init_debugfs(); rproc_init_cdev(); @@ -2550,6 +2559,8 @@ static void __exit remoteproc_exit(void) rproc_exit_panic(); rproc_exit_debugfs(); rproc_exit_sysfs(); + if (rproc_recovery_wq) + destroy_workqueue(rproc_recovery_wq); } module_exit(remoteproc_exit); diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 27d7bd1c9648..5572a1b7c0e5 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -290,7 +290,7 @@ u32 dwc3_core_fifo_space(struct dwc3_ep *dep, u8 type) * dwc3_core_soft_reset - Issues core soft reset and PHY reset * @dwc: pointer to our context structure */ -static int dwc3_core_soft_reset(struct dwc3 *dwc) +int dwc3_core_soft_reset(struct dwc3 *dwc) { u32 reg; int retries = 1000; diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 8701bb8a4b2a..45acc88e2137 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -732,6 +732,7 @@ struct dwc3_ep { #define DWC3_EP_FIRST_STREAM_PRIMED BIT(10) #define DWC3_EP_PENDING_CLEAR_STALL BIT(11) #define DWC3_EP_TXFIFO_RESIZED BIT(12) +#define DWC3_EP_DELAY_STOP BIT(13) /* This last one is specific to EP0 */ #define DWC3_EP0_DIR_IN BIT(31) @@ -1549,6 +1550,8 @@ bool dwc3_has_imod(struct dwc3 *dwc); int dwc3_event_buffers_setup(struct dwc3 *dwc); void dwc3_event_buffers_cleanup(struct dwc3 *dwc); +int dwc3_core_soft_reset(struct dwc3 *dwc); + #if IS_ENABLED(CONFIG_USB_DWC3_HOST) || IS_ENABLED(CONFIG_USB_DWC3_DUAL_ROLE) int dwc3_host_init(struct dwc3 *dwc); void dwc3_host_exit(struct dwc3 *dwc); diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 7b5190082aa8..5523a01ab1bc 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -271,6 +271,7 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) { struct dwc3_ep *dep; int ret; + int i; complete(&dwc->ep0_in_setup); @@ -279,6 +280,19 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) DWC3_TRBCTL_CONTROL_SETUP, false); ret = dwc3_ep0_start_trans(dep); WARN_ON(ret < 0); + for (i = 2; i < DWC3_ENDPOINTS_NUM; i++) { + struct dwc3_ep *dwc3_ep; + + dwc3_ep = dwc->eps[i]; + if (!dwc3_ep) + continue; + + if (!(dwc3_ep->flags & DWC3_EP_DELAY_STOP)) + continue; + + dwc3_ep->flags &= ~DWC3_EP_DELAY_STOP; + dwc3_stop_active_transfer(dwc3_ep, true, true); + } } static struct dwc3_ep *dwc3_wIndex_to_dep(struct dwc3 *dwc, __le16 wIndex_le) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index c27905c0a004..0866d92302bb 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -2015,6 +2015,7 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) */ if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || (dep->flags & DWC3_EP_WEDGE) || + (dep->flags & DWC3_EP_DELAY_STOP) || (dep->flags & DWC3_EP_STALL)) { dep->flags |= DWC3_EP_DELAY_START; return 0; @@ -2094,10 +2095,10 @@ static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *r static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) { struct dwc3_request *req; - struct dwc3_request *tmp; struct dwc3 *dwc = dep->dwc; - list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) { + while (!list_empty(&dep->cancelled_list)) { + req = next_request(&dep->cancelled_list); dwc3_gadget_ep_skip_trbs(dep, req); switch (req->status) { case DWC3_REQUEST_STATUS_DISCONNECTED: @@ -2114,6 +2115,12 @@ static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) dwc3_gadget_giveback(dep, req, -ECONNRESET); break; } + /* + * The endpoint is disabled, let the dwc3_remove_requests() + * handle the cleanup. + */ + if (!dep->endpoint.desc) + break; } } @@ -2148,6 +2155,16 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, list_for_each_entry(r, &dep->started_list, list) { if (r == req) { + /* + * If a Setup packet is received but yet to DMA out, the controller will + * not process the End Transfer command of any endpoint. Polling of its + * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a + * timeout. Delay issuing the End Transfer command until the Setup TRB is + * prepared. + */ + if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) + dep->flags |= DWC3_EP_DELAY_STOP; + /* wait until it is processed */ dwc3_stop_active_transfer(dep, true, true); @@ -2232,7 +2249,8 @@ int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) if (!list_empty(&dep->started_list)) dep->flags |= DWC3_EP_DELAY_START; - if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) { + if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || + (dep->flags & DWC3_EP_DELAY_STOP)) { dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; return 0; } @@ -2664,6 +2682,17 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) dwc->ev_buf->length; } } else { + /* + * In the Synopsys DWC_usb31 1.90a programming guide section + * 4.1.9, it specifies that for a reconnect after a + * device-initiated disconnect requires a core soft reset + * (DCTL.CSftRst) before enabling the run/stop bit. + */ + spin_unlock_irqrestore(&dwc->lock, flags); + dwc3_core_soft_reset(dwc); + spin_lock_irqsave(&dwc->lock, flags); + + dwc3_event_buffers_setup(dwc); __dwc3_gadget_start(dwc); } @@ -3443,15 +3472,21 @@ static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, const struct dwc3_event_depevt *event, int status) { struct dwc3_request *req; - struct dwc3_request *tmp; - list_for_each_entry_safe(req, tmp, &dep->started_list, list) { + while (!list_empty(&dep->started_list)) { int ret; + req = next_request(&dep->started_list); ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, req, status); if (ret) break; + /* + * The endpoint is disabled, let the dwc3_remove_requests() + * handle the cleanup. + */ + if (!dep->endpoint.desc) + break; } } @@ -3811,6 +3846,7 @@ void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, int ret; if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || + (dep->flags & DWC3_EP_DELAY_STOP) || (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) return; diff --git a/drivers/usb/dwc3/gadget.h b/drivers/usb/dwc3/gadget.h index 8129beb07466..e893c936b49d 100644 --- a/drivers/usb/dwc3/gadget.h +++ b/drivers/usb/dwc3/gadget.h @@ -132,6 +132,7 @@ int dwc3_gadget_ep0_queue(struct usb_ep *ep, struct usb_request *request, gfp_t gfp_flags); int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol); void dwc3_ep0_send_delayed_status(struct dwc3 *dwc); +void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt); /** * dwc3_gadget_ep_get_transfer_index - Gets transfer index from HW diff --git a/fs/crypto/Makefile b/fs/crypto/Makefile index 652c7180ec6d..d39077502e06 100644 --- a/fs/crypto/Makefile +++ b/fs/crypto/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_FS_ENCRYPTION) += fscrypto.o +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + fscrypto-y := crypto.o \ fname.o \ hkdf.o \ diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index c0b679216727..a6b3a5553fde 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -2036,6 +2036,15 @@ static int ext4_writepage(struct page *page, else len = PAGE_SIZE; + /* Should never happen but for bugs in other kernel subsystems */ + if (!page_has_buffers(page)) { + ext4_warning_inode(inode, + "page %lu does not have buffers attached", page->index); + ClearPageDirty(page); + unlock_page(page); + return 0; + } + page_bufs = page_buffers(page); /* * We cannot do block allocation or other extent handling in this @@ -2639,6 +2648,22 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) wait_on_page_writeback(page); BUG_ON(PageWriteback(page)); + /* + * Should never happen but for buggy code in + * other subsystems that call + * set_page_dirty() without properly warning + * the file system first. See [1] for more + * information. + * + * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz + */ + if (!page_has_buffers(page)) { + ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", page->index); + ClearPageDirty(page); + unlock_page(page); + continue; + } + if (mpd->map.m_len == 0) mpd->first_page = page->index; mpd->next_page = page->index + 1; diff --git a/fs/fs_types.c b/fs/fs_types.c index 78365e5dc08c..a11a1d8c7811 100644 --- a/fs/fs_types.c +++ b/fs/fs_types.c @@ -41,7 +41,7 @@ unsigned char fs_ftype_to_dtype(unsigned int filetype) return fs_dtype_by_ftype[filetype]; } -EXPORT_SYMBOL_GPL(fs_ftype_to_dtype); +EXPORT_SYMBOL_NS_GPL(fs_ftype_to_dtype, ANDROID_GKI_VFS_EXPORT_ONLY); /* * dirent file type to fs on-disk file type conversion diff --git a/fs/fscache/Makefile b/fs/fscache/Makefile index 79e08e05ef84..9121382cf16d 100644 --- a/fs/fscache/Makefile +++ b/fs/fscache/Makefile @@ -3,6 +3,8 @@ # Makefile for general filesystem caching code # +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + fscache-y := \ cache.o \ cookie.o \ diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 020e025f8022..42e8bcc011f0 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -950,7 +950,17 @@ static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep, while (count) { if (cs->write && cs->pipebufs && page) { - return fuse_ref_page(cs, page, offset, count); + /* + * Can't control lifetime of pipe buffers, so always + * copy user pages. + */ + if (cs->req->args->user_pages) { + err = fuse_copy_fill(cs); + if (err) + return err; + } else { + return fuse_ref_page(cs, page, offset, count); + } } else if (!cs->len) { if (cs->move_pages && page && offset == 0 && count == PAGE_SIZE) { diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 97a40476d1bd..a68c8310942f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1420,6 +1420,7 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, (PAGE_SIZE - ret) & (PAGE_SIZE - 1); } + ap->args.user_pages = true; if (write) ap->args.in_pages = true; else diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index fc5fcd8e0671..69a631c12b15 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -277,6 +277,7 @@ struct fuse_args { bool nocreds:1; bool in_pages:1; bool out_pages:1; + bool user_pages:1; bool out_argvar:1; bool page_zeroing:1; bool page_replace:1; diff --git a/fs/incfs/data_mgmt.h b/fs/incfs/data_mgmt.h index 84db0382765d..2227913674a4 100644 --- a/fs/incfs/data_mgmt.h +++ b/fs/incfs/data_mgmt.h @@ -131,8 +131,14 @@ struct mount_info { struct path mi_backing_dir_path; struct dentry *mi_index_dir; + /* For stacking mounts, if true, this indicates if the index dir needs + * to be freed for this SB otherwise it was created by lower level SB */ + bool mi_index_free; struct dentry *mi_incomplete_dir; + /* For stacking mounts, if true, this indicates if the incomplete dir + * needs to be freed for this SB. Similar to mi_index_free */ + bool mi_incomplete_free; const struct cred *mi_owner; diff --git a/fs/incfs/vfs.c b/fs/incfs/vfs.c index ea7866fbfd6e..776640451f6f 100644 --- a/fs/incfs/vfs.c +++ b/fs/incfs/vfs.c @@ -437,7 +437,8 @@ static int incfs_init_dentry(struct dentry *dentry, struct path *path) } static struct dentry *open_or_create_special_dir(struct dentry *backing_dir, - const char *name) + const char *name, + bool *created) { struct dentry *index_dentry; struct inode *backing_inode = d_inode(backing_dir); @@ -450,6 +451,7 @@ static struct dentry *open_or_create_special_dir(struct dentry *backing_dir, return index_dentry; } else if (d_really_is_positive(index_dentry)) { /* Index already exists. */ + *created = false; return index_dentry; } @@ -469,6 +471,7 @@ static struct dentry *open_or_create_special_dir(struct dentry *backing_dir, return ERR_PTR(-EINVAL); } + *created = true; return index_dentry; } @@ -1747,6 +1750,7 @@ struct dentry *incfs_mount_fs(struct file_system_type *type, int flags, struct super_block *src_fs_sb = NULL; struct inode *root_inode = NULL; struct super_block *sb = sget(type, NULL, set_anon_super, flags, NULL); + bool dir_created = false; int error = 0; if (IS_ERR(sb)) @@ -1763,17 +1767,23 @@ struct dentry *incfs_mount_fs(struct file_system_type *type, int flags, BUILD_BUG_ON(PAGE_SIZE != INCFS_DATA_FILE_BLOCK_SIZE); + if (!dev_name) { + pr_err("incfs: Backing dir is not set, filesystem can't be mounted.\n"); + error = -ENOENT; + goto err_deactivate; + } + error = parse_options(&options, (char *)data); if (error != 0) { pr_err("incfs: Options parsing error. %d\n", error); - goto err; + goto err_deactivate; } sb->s_bdi->ra_pages = options.readahead_pages; if (!dev_name) { pr_err("incfs: Backing dir is not set, filesystem can't be mounted.\n"); error = -ENOENT; - goto err; + goto err_free_opts; } error = kern_path(dev_name, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, @@ -1782,69 +1792,80 @@ struct dentry *incfs_mount_fs(struct file_system_type *type, int flags, !d_really_is_positive(backing_dir_path.dentry)) { pr_err("incfs: Error accessing: %s.\n", dev_name); - goto err; + goto err_free_opts; } src_fs_sb = backing_dir_path.dentry->d_sb; sb->s_maxbytes = src_fs_sb->s_maxbytes; + sb->s_stack_depth = src_fs_sb->s_stack_depth + 1; + + if (sb->s_stack_depth > FILESYSTEM_MAX_STACK_DEPTH) { + error = -EINVAL; + goto err_put_path; + } mi = incfs_alloc_mount_info(sb, &options, &backing_dir_path); - if (IS_ERR_OR_NULL(mi)) { error = PTR_ERR(mi); pr_err("incfs: Error allocating mount info. %d\n", error); - mi = NULL; - goto err; + goto err_put_path; } + sb->s_fs_info = mi; + mi->mi_backing_dir_path = backing_dir_path; index_dir = open_or_create_special_dir(backing_dir_path.dentry, - INCFS_INDEX_NAME); + INCFS_INDEX_NAME, &dir_created); if (IS_ERR_OR_NULL(index_dir)) { error = PTR_ERR(index_dir); pr_err("incfs: Can't find or create .index dir in %s\n", dev_name); /* No need to null index_dir since we don't put it */ - goto err; + goto err_put_path; } + mi->mi_index_dir = index_dir; + mi->mi_index_free = dir_created; incomplete_dir = open_or_create_special_dir(backing_dir_path.dentry, - INCFS_INCOMPLETE_NAME); + INCFS_INCOMPLETE_NAME, + &dir_created); if (IS_ERR_OR_NULL(incomplete_dir)) { error = PTR_ERR(incomplete_dir); pr_err("incfs: Can't find or create .incomplete dir in %s\n", dev_name); /* No need to null incomplete_dir since we don't put it */ - goto err; + goto err_put_path; } mi->mi_incomplete_dir = incomplete_dir; + mi->mi_incomplete_free = dir_created; - sb->s_fs_info = mi; root_inode = fetch_regular_inode(sb, backing_dir_path.dentry); if (IS_ERR(root_inode)) { error = PTR_ERR(root_inode); - goto err; + goto err_put_path; } sb->s_root = d_make_root(root_inode); if (!sb->s_root) { error = -ENOMEM; - goto err; + goto err_put_path; } error = incfs_init_dentry(sb->s_root, &backing_dir_path); if (error) - goto err; + goto err_put_path; path_put(&backing_dir_path); sb->s_flags |= SB_ACTIVE; pr_debug("incfs: mount\n"); return dget(sb->s_root); -err: - sb->s_fs_info = NULL; + +err_put_path: path_put(&backing_dir_path); - incfs_free_mount_info(mi); - deactivate_locked_super(sb); +err_free_opts: free_options(&options); +err_deactivate: + deactivate_locked_super(sb); + pr_err("incfs: mount failed %d\n", error); return ERR_PTR(error); } @@ -1879,10 +1900,26 @@ static int incfs_remount_fs(struct super_block *sb, int *flags, char *data) void incfs_kill_sb(struct super_block *sb) { struct mount_info *mi = sb->s_fs_info; + struct inode *dinode = NULL; pr_debug("incfs: unmount\n"); - generic_shutdown_super(sb); - incfs_free_mount_info(mi); + + if (mi) { + if (mi->mi_backing_dir_path.dentry) + dinode = d_inode(mi->mi_backing_dir_path.dentry); + + if (dinode) { + if (mi->mi_index_dir && mi->mi_index_free) + vfs_rmdir(dinode, mi->mi_index_dir); + + if (mi->mi_incomplete_dir && mi->mi_incomplete_free) + vfs_rmdir(dinode, mi->mi_incomplete_dir); + } + + incfs_free_mount_info(mi); + sb->s_fs_info = NULL; + } + kill_anon_super(sb); } static int show_options(struct seq_file *m, struct dentry *root) diff --git a/fs/jbd2/Makefile b/fs/jbd2/Makefile index 126b4da6c7de..b64f93331643 100644 --- a/fs/jbd2/Makefile +++ b/fs/jbd2/Makefile @@ -3,6 +3,8 @@ # Makefile for the linux journaling routines. # +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + obj-$(CONFIG_JBD2) += jbd2.o jbd2-objs := transaction.o commit.o recovery.o checkpoint.o revoke.o journal.o diff --git a/fs/verity/Makefile b/fs/verity/Makefile index 435559a4fa9e..4b8323450f90 100644 --- a/fs/verity/Makefile +++ b/fs/verity/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=ANDROID_GKI_VFS_EXPORT_ONLY + obj-$(CONFIG_FS_VERITY) += enable.o \ hash_algs.o \ init.o \ diff --git a/include/linux/android_kabi.h b/include/linux/android_kabi.h index dc0da1ab45d6..f6dd7f00b386 100644 --- a/include/linux/android_kabi.h +++ b/include/linux/android_kabi.h @@ -83,7 +83,7 @@ * number: the "number" of the padding variable in the structure. Start with * 1 and go up. */ -#ifdef CONFIG_ANDROID_STRUCT_PADDING +#ifdef CONFIG_ANDROID_KABI_RESERVE #define ANDROID_KABI_RESERVE(number) _ANDROID_KABI_RESERVE(number) #else #define ANDROID_KABI_RESERVE(number) diff --git a/include/linux/android_vendor.h b/include/linux/android_vendor.h index ab3dca23966a..af3014ccc82e 100644 --- a/include/linux/android_vendor.h +++ b/include/linux/android_vendor.h @@ -26,7 +26,7 @@ * Same as ANDROID_VENDOR_DATA but allocates an array of u64 with * the specified size */ -#ifdef CONFIG_ANDROID_STRUCT_PADDING +#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA #define ANDROID_VENDOR_DATA(n) u64 android_vendor_data##n #define ANDROID_VENDOR_DATA_ARRAY(n, s) u64 android_vendor_data##n[s] diff --git a/include/linux/damon.h b/include/linux/damon.h new file mode 100644 index 000000000000..5e1e3a128b77 --- /dev/null +++ b/include/linux/damon.h @@ -0,0 +1,511 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DAMON api + * + * Author: SeongJae Park + */ + +#ifndef _DAMON_H_ +#define _DAMON_H_ + +#include +#include +#include +#include + +/* Minimal region size. Every damon_region is aligned by this. */ +#define DAMON_MIN_REGION PAGE_SIZE +/* Max priority score for DAMON-based operation schemes */ +#define DAMOS_MAX_SCORE (99) + +/* Get a random number in [l, r) */ +static inline unsigned long damon_rand(unsigned long l, unsigned long r) +{ + return l + prandom_u32_max(r - l); +} + +/** + * struct damon_addr_range - Represents an address region of [@start, @end). + * @start: Start address of the region (inclusive). + * @end: End address of the region (exclusive). + */ +struct damon_addr_range { + unsigned long start; + unsigned long end; +}; + +/** + * struct damon_region - Represents a monitoring target region. + * @ar: The address range of the region. + * @sampling_addr: Address of the sample for the next access check. + * @nr_accesses: Access frequency of this region. + * @list: List head for siblings. + * @age: Age of this region. + * + * @age is initially zero, increased for each aggregation interval, and reset + * to zero again if the access frequency is significantly changed. If two + * regions are merged into a new region, both @nr_accesses and @age of the new + * region are set as region size-weighted average of those of the two regions. + */ +struct damon_region { + struct damon_addr_range ar; + unsigned long sampling_addr; + unsigned int nr_accesses; + struct list_head list; + + unsigned int age; +/* private: Internal value for age calculation. */ + unsigned int last_nr_accesses; +}; + +/** + * struct damon_target - Represents a monitoring target. + * @id: Unique identifier for this target. + * @nr_regions: Number of monitoring target regions of this target. + * @regions_list: Head of the monitoring target regions of this target. + * @list: List head for siblings. + * + * Each monitoring context could have multiple targets. For example, a context + * for virtual memory address spaces could have multiple target processes. The + * @id of each target should be unique among the targets of the context. For + * example, in the virtual address monitoring context, it could be a pidfd or + * an address of an mm_struct. + */ +struct damon_target { + unsigned long id; + unsigned int nr_regions; + struct list_head regions_list; + struct list_head list; +}; + +/** + * enum damos_action - Represents an action of a Data Access Monitoring-based + * Operation Scheme. + * + * @DAMOS_WILLNEED: Call ``madvise()`` for the region with MADV_WILLNEED. + * @DAMOS_COLD: Call ``madvise()`` for the region with MADV_COLD. + * @DAMOS_PAGEOUT: Call ``madvise()`` for the region with MADV_PAGEOUT. + * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. + * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. + * @DAMOS_STAT: Do nothing but count the stat. + */ +enum damos_action { + DAMOS_WILLNEED, + DAMOS_COLD, + DAMOS_PAGEOUT, + DAMOS_HUGEPAGE, + DAMOS_NOHUGEPAGE, + DAMOS_STAT, /* Do nothing but only record the stat */ +}; + +/** + * struct damos_quota - Controls the aggressiveness of the given scheme. + * @ms: Maximum milliseconds that the scheme can use. + * @sz: Maximum bytes of memory that the action can be applied. + * @reset_interval: Charge reset interval in milliseconds. + * + * @weight_sz: Weight of the region's size for prioritization. + * @weight_nr_accesses: Weight of the region's nr_accesses for prioritization. + * @weight_age: Weight of the region's age for prioritization. + * + * To avoid consuming too much CPU time or IO resources for applying the + * &struct damos->action to large memory, DAMON allows users to set time and/or + * size quotas. The quotas can be set by writing non-zero values to &ms and + * &sz, respectively. If the time quota is set, DAMON tries to use only up to + * &ms milliseconds within &reset_interval for applying the action. If the + * size quota is set, DAMON tries to apply the action only up to &sz bytes + * within &reset_interval. + * + * Internally, the time quota is transformed to a size quota using estimated + * throughput of the scheme's action. DAMON then compares it against &sz and + * uses smaller one as the effective quota. + * + * For selecting regions within the quota, DAMON prioritizes current scheme's + * target memory regions using the &struct damon_primitive->get_scheme_score. + * You could customize the prioritization logic by setting &weight_sz, + * &weight_nr_accesses, and &weight_age, because monitoring primitives are + * encouraged to respect those. + */ +struct damos_quota { + unsigned long ms; + unsigned long sz; + unsigned long reset_interval; + + unsigned int weight_sz; + unsigned int weight_nr_accesses; + unsigned int weight_age; + +/* private: */ + /* For throughput estimation */ + unsigned long total_charged_sz; + unsigned long total_charged_ns; + + unsigned long esz; /* Effective size quota in bytes */ + + /* For charging the quota */ + unsigned long charged_sz; + unsigned long charged_from; + struct damon_target *charge_target_from; + unsigned long charge_addr_from; + + /* For prioritization */ + unsigned long histogram[DAMOS_MAX_SCORE + 1]; + unsigned int min_score; +}; + +/** + * enum damos_wmark_metric - Represents the watermark metric. + * + * @DAMOS_WMARK_NONE: Ignore the watermarks of the given scheme. + * @DAMOS_WMARK_FREE_MEM_RATE: Free memory rate of the system in [0,1000]. + */ +enum damos_wmark_metric { + DAMOS_WMARK_NONE, + DAMOS_WMARK_FREE_MEM_RATE, +}; + +/** + * struct damos_watermarks - Controls when a given scheme should be activated. + * @metric: Metric for the watermarks. + * @interval: Watermarks check time interval in microseconds. + * @high: High watermark. + * @mid: Middle watermark. + * @low: Low watermark. + * + * If &metric is &DAMOS_WMARK_NONE, the scheme is always active. Being active + * means DAMON does monitoring and applying the action of the scheme to + * appropriate memory regions. Else, DAMON checks &metric of the system for at + * least every &interval microseconds and works as below. + * + * If &metric is higher than &high, the scheme is inactivated. If &metric is + * between &mid and &low, the scheme is activated. If &metric is lower than + * &low, the scheme is inactivated. + */ +struct damos_watermarks { + enum damos_wmark_metric metric; + unsigned long interval; + unsigned long high; + unsigned long mid; + unsigned long low; + +/* private: */ + bool activated; +}; + +/** + * struct damos_stat - Statistics on a given scheme. + * @nr_tried: Total number of regions that the scheme is tried to be applied. + * @sz_tried: Total size of regions that the scheme is tried to be applied. + * @nr_applied: Total number of regions that the scheme is applied. + * @sz_applied: Total size of regions that the scheme is applied. + * @qt_exceeds: Total number of times the quota of the scheme has exceeded. + */ +struct damos_stat { + unsigned long nr_tried; + unsigned long sz_tried; + unsigned long nr_applied; + unsigned long sz_applied; + unsigned long qt_exceeds; +}; + +/** + * struct damos - Represents a Data Access Monitoring-based Operation Scheme. + * @min_sz_region: Minimum size of target regions. + * @max_sz_region: Maximum size of target regions. + * @min_nr_accesses: Minimum ``->nr_accesses`` of target regions. + * @max_nr_accesses: Maximum ``->nr_accesses`` of target regions. + * @min_age_region: Minimum age of target regions. + * @max_age_region: Maximum age of target regions. + * @action: &damo_action to be applied to the target regions. + * @quota: Control the aggressiveness of this scheme. + * @wmarks: Watermarks for automated (in)activation of this scheme. + * @stat: Statistics of this scheme. + * @list: List head for siblings. + * + * For each aggregation interval, DAMON finds regions which fit in the + * condition (&min_sz_region, &max_sz_region, &min_nr_accesses, + * &max_nr_accesses, &min_age_region, &max_age_region) and applies &action to + * those. To avoid consuming too much CPU time or IO resources for the + * &action, "a is used. + * + * To do the work only when needed, schemes can be activated for specific + * system situations using &wmarks. If all schemes that registered to the + * monitoring context are inactive, DAMON stops monitoring either, and just + * repeatedly checks the watermarks. + * + * If all schemes that registered to a &struct damon_ctx are inactive, DAMON + * stops monitoring and just repeatedly checks the watermarks. + * + * After applying the &action to each region, &stat_count and &stat_sz is + * updated to reflect the number of regions and total size of regions that the + * &action is applied. + */ +struct damos { + unsigned long min_sz_region; + unsigned long max_sz_region; + unsigned int min_nr_accesses; + unsigned int max_nr_accesses; + unsigned int min_age_region; + unsigned int max_age_region; + enum damos_action action; + struct damos_quota quota; + struct damos_watermarks wmarks; + struct damos_stat stat; + struct list_head list; +}; + +struct damon_ctx; + +/** + * struct damon_primitive - Monitoring primitives for given use cases. + * + * @init: Initialize primitive-internal data structures. + * @update: Update primitive-internal data structures. + * @prepare_access_checks: Prepare next access check of target regions. + * @check_accesses: Check the accesses to target regions. + * @reset_aggregated: Reset aggregated accesses monitoring results. + * @get_scheme_score: Get the score of a region for a scheme. + * @apply_scheme: Apply a DAMON-based operation scheme. + * @target_valid: Determine if the target is valid. + * @cleanup: Clean up the context. + * + * DAMON can be extended for various address spaces and usages. For this, + * users should register the low level primitives for their target address + * space and usecase via the &damon_ctx.primitive. Then, the monitoring thread + * (&damon_ctx.kdamond) calls @init and @prepare_access_checks before starting + * the monitoring, @update after each &damon_ctx.primitive_update_interval, and + * @check_accesses, @target_valid and @prepare_access_checks after each + * &damon_ctx.sample_interval. Finally, @reset_aggregated is called after each + * &damon_ctx.aggr_interval. + * + * @init should initialize primitive-internal data structures. For example, + * this could be used to construct proper monitoring target regions and link + * those to @damon_ctx.adaptive_targets. + * @update should update the primitive-internal data structures. For example, + * this could be used to update monitoring target regions for current status. + * @prepare_access_checks should manipulate the monitoring regions to be + * prepared for the next access check. + * @check_accesses should check the accesses to each region that made after the + * last preparation and update the number of observed accesses of each region. + * It should also return max number of observed accesses that made as a result + * of its update. The value will be used for regions adjustment threshold. + * @reset_aggregated should reset the access monitoring results that aggregated + * by @check_accesses. + * @get_scheme_score should return the priority score of a region for a scheme + * as an integer in [0, &DAMOS_MAX_SCORE]. + * @apply_scheme is called from @kdamond when a region for user provided + * DAMON-based operation scheme is found. It should apply the scheme's action + * to the region and return bytes of the region that the action is successfully + * applied. + * @target_valid should check whether the target is still valid for the + * monitoring. + * @cleanup is called from @kdamond just before its termination. + */ +struct damon_primitive { + void (*init)(struct damon_ctx *context); + void (*update)(struct damon_ctx *context); + void (*prepare_access_checks)(struct damon_ctx *context); + unsigned int (*check_accesses)(struct damon_ctx *context); + void (*reset_aggregated)(struct damon_ctx *context); + int (*get_scheme_score)(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme); + unsigned long (*apply_scheme)(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme); + bool (*target_valid)(void *target); + void (*cleanup)(struct damon_ctx *context); +}; + +/** + * struct damon_callback - Monitoring events notification callbacks. + * + * @before_start: Called before starting the monitoring. + * @after_sampling: Called after each sampling. + * @after_aggregation: Called after each aggregation. + * @before_terminate: Called before terminating the monitoring. + * @private: User private data. + * + * The monitoring thread (&damon_ctx.kdamond) calls @before_start and + * @before_terminate just before starting and finishing the monitoring, + * respectively. Therefore, those are good places for installing and cleaning + * @private. + * + * The monitoring thread calls @after_sampling and @after_aggregation for each + * of the sampling intervals and aggregation intervals, respectively. + * Therefore, users can safely access the monitoring results without additional + * protection. For the reason, users are recommended to use these callback for + * the accesses to the results. + * + * If any callback returns non-zero, monitoring stops. + */ +struct damon_callback { + void *private; + + int (*before_start)(struct damon_ctx *context); + int (*after_sampling)(struct damon_ctx *context); + int (*after_aggregation)(struct damon_ctx *context); + void (*before_terminate)(struct damon_ctx *context); +}; + +/** + * struct damon_ctx - Represents a context for each monitoring. This is the + * main interface that allows users to set the attributes and get the results + * of the monitoring. + * + * @sample_interval: The time between access samplings. + * @aggr_interval: The time between monitor results aggregations. + * @primitive_update_interval: The time between monitoring primitive updates. + * + * For each @sample_interval, DAMON checks whether each region is accessed or + * not. It aggregates and keeps the access information (number of accesses to + * each region) for @aggr_interval time. DAMON also checks whether the target + * memory regions need update (e.g., by ``mmap()`` calls from the application, + * in case of virtual memory monitoring) and applies the changes for each + * @primitive_update_interval. All time intervals are in micro-seconds. + * Please refer to &struct damon_primitive and &struct damon_callback for more + * detail. + * + * @kdamond: Kernel thread who does the monitoring. + * @kdamond_stop: Notifies whether kdamond should stop. + * @kdamond_lock: Mutex for the synchronizations with @kdamond. + * + * For each monitoring context, one kernel thread for the monitoring is + * created. The pointer to the thread is stored in @kdamond. + * + * Once started, the monitoring thread runs until explicitly required to be + * terminated or every monitoring target is invalid. The validity of the + * targets is checked via the &damon_primitive.target_valid of @primitive. The + * termination can also be explicitly requested by writing non-zero to + * @kdamond_stop. The thread sets @kdamond to NULL when it terminates. + * Therefore, users can know whether the monitoring is ongoing or terminated by + * reading @kdamond. Reads and writes to @kdamond and @kdamond_stop from + * outside of the monitoring thread must be protected by @kdamond_lock. + * + * Note that the monitoring thread protects only @kdamond and @kdamond_stop via + * @kdamond_lock. Accesses to other fields must be protected by themselves. + * + * @primitive: Set of monitoring primitives for given use cases. + * @callback: Set of callbacks for monitoring events notifications. + * + * @min_nr_regions: The minimum number of adaptive monitoring regions. + * @max_nr_regions: The maximum number of adaptive monitoring regions. + * @adaptive_targets: Head of monitoring targets (&damon_target) list. + * @schemes: Head of schemes (&damos) list. + */ +struct damon_ctx { + unsigned long sample_interval; + unsigned long aggr_interval; + unsigned long primitive_update_interval; + +/* private: internal use only */ + struct timespec64 last_aggregation; + struct timespec64 last_primitive_update; + +/* public: */ + struct task_struct *kdamond; + struct mutex kdamond_lock; + + struct damon_primitive primitive; + struct damon_callback callback; + + unsigned long min_nr_regions; + unsigned long max_nr_regions; + struct list_head adaptive_targets; + struct list_head schemes; +}; + +static inline struct damon_region *damon_next_region(struct damon_region *r) +{ + return container_of(r->list.next, struct damon_region, list); +} + +static inline struct damon_region *damon_prev_region(struct damon_region *r) +{ + return container_of(r->list.prev, struct damon_region, list); +} + +static inline struct damon_region *damon_last_region(struct damon_target *t) +{ + return list_last_entry(&t->regions_list, struct damon_region, list); +} + +#define damon_for_each_region(r, t) \ + list_for_each_entry(r, &t->regions_list, list) + +#define damon_for_each_region_safe(r, next, t) \ + list_for_each_entry_safe(r, next, &t->regions_list, list) + +#define damon_for_each_target(t, ctx) \ + list_for_each_entry(t, &(ctx)->adaptive_targets, list) + +#define damon_for_each_target_safe(t, next, ctx) \ + list_for_each_entry_safe(t, next, &(ctx)->adaptive_targets, list) + +#define damon_for_each_scheme(s, ctx) \ + list_for_each_entry(s, &(ctx)->schemes, list) + +#define damon_for_each_scheme_safe(s, next, ctx) \ + list_for_each_entry_safe(s, next, &(ctx)->schemes, list) + +#ifdef CONFIG_DAMON + +struct damon_region *damon_new_region(unsigned long start, unsigned long end); + +/* + * Add a region between two other regions + */ +static inline void damon_insert_region(struct damon_region *r, + struct damon_region *prev, struct damon_region *next, + struct damon_target *t) +{ + __list_add(&r->list, &prev->list, &next->list); + t->nr_regions++; +} + +void damon_add_region(struct damon_region *r, struct damon_target *t); +void damon_destroy_region(struct damon_region *r, struct damon_target *t); + +struct damos *damon_new_scheme( + unsigned long min_sz_region, unsigned long max_sz_region, + unsigned int min_nr_accesses, unsigned int max_nr_accesses, + unsigned int min_age_region, unsigned int max_age_region, + enum damos_action action, struct damos_quota *quota, + struct damos_watermarks *wmarks); +void damon_add_scheme(struct damon_ctx *ctx, struct damos *s); +void damon_destroy_scheme(struct damos *s); + +struct damon_target *damon_new_target(unsigned long id); +void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); +bool damon_targets_empty(struct damon_ctx *ctx); +void damon_free_target(struct damon_target *t); +void damon_destroy_target(struct damon_target *t); +unsigned int damon_nr_regions(struct damon_target *t); + +struct damon_ctx *damon_new_ctx(void); +void damon_destroy_ctx(struct damon_ctx *ctx); +int damon_set_targets(struct damon_ctx *ctx, + unsigned long *ids, ssize_t nr_ids); +int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, + unsigned long aggr_int, unsigned long primitive_upd_int, + unsigned long min_nr_reg, unsigned long max_nr_reg); +int damon_set_schemes(struct damon_ctx *ctx, + struct damos **schemes, ssize_t nr_schemes); +int damon_nr_running_ctxs(void); + +int damon_start(struct damon_ctx **ctxs, int nr_ctxs); +int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); + +#endif /* CONFIG_DAMON */ + +#ifdef CONFIG_DAMON_VADDR +bool damon_va_target_valid(void *t); +void damon_va_set_primitives(struct damon_ctx *ctx); +#endif /* CONFIG_DAMON_VADDR */ + +#ifdef CONFIG_DAMON_PADDR +bool damon_pa_target_valid(void *t); +void damon_pa_set_primitives(struct damon_ctx *ctx); +#endif /* CONFIG_DAMON_PADDR */ + +#endif /* _DAMON_H */ diff --git a/include/linux/delay.h b/include/linux/delay.h index 1d0e2ce6b6d9..abecbccab6e4 100644 --- a/include/linux/delay.h +++ b/include/linux/delay.h @@ -20,6 +20,7 @@ */ #include +#include extern unsigned long loops_per_jiffy; @@ -58,8 +59,15 @@ void calibrate_delay(void); void __attribute__((weak)) calibration_delay_done(void); void msleep(unsigned int msecs); unsigned long msleep_interruptible(unsigned int msecs); +void usleep_range_state(unsigned long min, unsigned long max, + unsigned int state); void usleep_range(unsigned long min, unsigned long max); +static inline void usleep_idle_range(unsigned long min, unsigned long max) +{ + usleep_range_state(min, max, TASK_IDLE); +} + static inline void ssleep(unsigned int seconds) { msleep(seconds * 1000); diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index a2738944340c..dc636f162194 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -132,7 +132,7 @@ enum pageflags { #ifdef CONFIG_MEMORY_FAILURE PG_hwpoison, /* hardware poisoned page. Don't touch */ #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) PG_young, PG_idle, #endif @@ -440,7 +440,7 @@ PAGEFLAG_FALSE(HWPoison) #define __PG_HWPOISON 0 #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) TESTPAGEFLAG(Young, young, PF_ANY) SETPAGEFLAG(Young, young, PF_ANY) TESTCLEARFLAG(Young, young, PF_ANY) diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index 31c429870dc8..bb9a4a2b2aa2 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -25,7 +25,7 @@ enum page_ext_flags { /* page migration failed */ PAGE_EXT_PINNER_MIGRATION_FAILED, #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) PAGE_EXT_YOUNG, PAGE_EXT_IDLE, #endif diff --git a/include/linux/page_idle.h b/include/linux/page_idle.h index 1e894d34bdce..d8a6aecf99cb 100644 --- a/include/linux/page_idle.h +++ b/include/linux/page_idle.h @@ -6,7 +6,7 @@ #include #include -#ifdef CONFIG_IDLE_PAGE_TRACKING +#ifdef CONFIG_PAGE_IDLE_FLAG #ifdef CONFIG_64BIT static inline bool page_is_young(struct page *page) @@ -106,7 +106,7 @@ static inline void clear_page_idle(struct page *page) } #endif /* CONFIG_64BIT */ -#else /* !CONFIG_IDLE_PAGE_TRACKING */ +#else /* !CONFIG_PAGE_IDLE_FLAG */ static inline bool page_is_young(struct page *page) { @@ -135,6 +135,6 @@ static inline void clear_page_idle(struct page *page) { } -#endif /* CONFIG_IDLE_PAGE_TRACKING */ +#endif /* CONFIG_PAGE_IDLE_FLAG */ #endif /* _LINUX_MM_PAGE_IDLE_H */ diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h new file mode 100644 index 000000000000..c79f1d4c39af --- /dev/null +++ b/include/trace/events/damon.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM damon + +#if !defined(_TRACE_DAMON_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DAMON_H + +#include +#include +#include + +TRACE_EVENT(damon_aggregated, + + TP_PROTO(struct damon_target *t, unsigned int target_id, + struct damon_region *r, unsigned int nr_regions), + + TP_ARGS(t, target_id, r, nr_regions), + + TP_STRUCT__entry( + __field(unsigned long, target_id) + __field(unsigned int, nr_regions) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned int, nr_accesses) + __field(unsigned int, age) + ), + + TP_fast_assign( + __entry->target_id = target_id; + __entry->nr_regions = nr_regions; + __entry->start = r->ar.start; + __entry->end = r->ar.end; + __entry->nr_accesses = r->nr_accesses; + __entry->age = r->age; + ), + + TP_printk("target_id=%lu nr_regions=%u %lu-%lu: %u %u", + __entry->target_id, __entry->nr_regions, + __entry->start, __entry->end, + __entry->nr_accesses, __entry->age) +); + +#endif /* _TRACE_DAMON_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 390270e00a1d..d428f0137c49 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -73,7 +73,7 @@ #define IF_HAVE_PG_HWPOISON(flag,string) #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && defined(CONFIG_64BIT) #define IF_HAVE_PG_IDLE(flag,string) ,{1UL << flag, string} #else #define IF_HAVE_PG_IDLE(flag,string) diff --git a/include/trace/hooks/mm.h b/include/trace/hooks/mm.h index bde2f3556c2a..4af456345aa4 100644 --- a/include/trace/hooks/mm.h +++ b/include/trace/hooks/mm.h @@ -132,6 +132,10 @@ struct device; DECLARE_HOOK(android_vh_subpage_dma_contig_alloc, TP_PROTO(bool *allow_subpage_alloc, struct device *dev, size_t *size), TP_ARGS(allow_subpage_alloc, dev, size)); +struct readahead_control; +DECLARE_HOOK(android_vh_ra_tuning_max_page, + TP_PROTO(struct readahead_control *ractl, unsigned long *max_page), + TP_ARGS(ractl, max_page)); /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_MM_H */ diff --git a/include/trace/hooks/rwsem.h b/include/trace/hooks/rwsem.h index d644a6e21259..c8e445216910 100644 --- a/include/trace/hooks/rwsem.h +++ b/include/trace/hooks/rwsem.h @@ -29,7 +29,21 @@ DECLARE_HOOK(android_vh_alter_rwsem_list_add, DECLARE_HOOK(android_vh_rwsem_wake_finish, TP_PROTO(struct rw_semaphore *sem), TP_ARGS(sem)); - +DECLARE_HOOK(android_vh_rwsem_set_owner, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_set_reader_owned, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_up_write_end, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_up_read_end, + TP_PROTO(struct rw_semaphore *sem), + TP_ARGS(sem)); +DECLARE_HOOK(android_vh_rwsem_mark_wake_readers, + TP_PROTO(struct rw_semaphore *sem, struct rwsem_waiter *waiter), + TP_ARGS(sem, waiter)); /* macro versions of hooks are no longer required */ #endif /* _TRACE_HOOK_RWSEM_H */ diff --git a/include/trace/hooks/vmscan.h b/include/trace/hooks/vmscan.h index 48a00c6cede0..7b4d222301af 100644 --- a/include/trace/hooks/vmscan.h +++ b/include/trace/hooks/vmscan.h @@ -31,6 +31,9 @@ DECLARE_HOOK(android_vh_page_referenced_check_bypass, DECLARE_HOOK(android_vh_shrink_node_memcgs, TP_PROTO(struct mem_cgroup *memcg, bool *skip), TP_ARGS(memcg, skip)); +DECLARE_HOOK(android_vh_tune_memcg_scan_type, + TP_PROTO(struct mem_cgroup *memcg, char *scan_type), + TP_ARGS(memcg, scan_type)); #endif /* _TRACE_HOOK_VMSCAN_H */ /* This part must be outside protection */ #include diff --git a/init/init_task.c b/init/init_task.c index 26e018fbbf78..9b8ad6861316 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -213,7 +213,7 @@ struct task_struct init_task #ifdef CONFIG_SECCOMP_FILTER .seccomp = { .filter_count = ATOMIC_INIT(0) }, #endif -#ifdef CONFIG_ANDROID_STRUCT_PADDING +#ifdef CONFIG_ANDROID_VENDOR_OEM_DATA .android_vendor_data1 = {0, }, .android_oem_data1 = {0, }, #endif diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 30a5f91f99cf..a33e6a36dae9 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -176,6 +176,7 @@ static inline void rwsem_set_owner(struct rw_semaphore *sem) { atomic_long_set(&sem->owner, (long)current); + trace_android_vh_rwsem_set_owner(sem); } static inline void rwsem_clear_owner(struct rw_semaphore *sem) @@ -213,6 +214,7 @@ static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) { __rwsem_set_reader_owned(sem, current); + trace_android_vh_rwsem_set_reader_owned(sem); } /* @@ -496,6 +498,7 @@ static void rwsem_mark_wake(struct rw_semaphore *sem, woken++; list_move_tail(&waiter->list, &wlist); + trace_android_vh_rwsem_mark_wake_readers(sem, waiter); /* * Limit # of readers that can be woken up per wakeup call. */ @@ -1460,6 +1463,7 @@ static inline void __up_read(struct rw_semaphore *sem) clear_wr_nonspinnable(sem); rwsem_wake(sem, tmp); } + trace_android_vh_rwsem_up_read_end(sem); } /* @@ -1481,6 +1485,7 @@ static inline void __up_write(struct rw_semaphore *sem) tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); if (unlikely(tmp & RWSEM_FLAG_WAITERS)) rwsem_wake(sem, tmp); + trace_android_vh_rwsem_up_write_end(sem); } /* diff --git a/kernel/time/timer.c b/kernel/time/timer.c index ed71c4141047..1535065a7d76 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2052,6 +2052,32 @@ unsigned long msleep_interruptible(unsigned int msecs) EXPORT_SYMBOL(msleep_interruptible); +/** + * usleep_range_state - Sleep for an approximate time in a given state + * @min: Minimum time in usecs to sleep + * @max: Maximum time in usecs to sleep + * @state: State of the current task that will be while sleeping + * + * In non-atomic context where the exact wakeup time is flexible, use + * usleep_range_state() instead of udelay(). The sleep improves responsiveness + * by avoiding the CPU-hogging busy-wait of udelay(), and the range reduces + * power usage by allowing hrtimers to take advantage of an already- + * scheduled interrupt instead of scheduling a new one just for this sleep. + */ +void __sched usleep_range_state(unsigned long min, unsigned long max, + unsigned int state) +{ + ktime_t exp = ktime_add_us(ktime_get(), min); + u64 delta = (u64)(max - min) * NSEC_PER_USEC; + + for (;;) { + __set_current_state(state); + /* Do not return before the requested sleep time has elapsed */ + if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS)) + break; + } +} + /** * usleep_range - Sleep for an approximate time * @min: Minimum time in usecs to sleep @@ -2065,14 +2091,6 @@ EXPORT_SYMBOL(msleep_interruptible); */ void __sched usleep_range(unsigned long min, unsigned long max) { - ktime_t exp = ktime_add_us(ktime_get(), min); - u64 delta = (u64)(max - min) * NSEC_PER_USEC; - - for (;;) { - __set_current_state(TASK_UNINTERRUPTIBLE); - /* Do not return before the requested sleep time has elapsed */ - if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS)) - break; - } + usleep_range_state(min, max, TASK_UNINTERRUPTIBLE); } EXPORT_SYMBOL(usleep_range); diff --git a/mm/Kconfig b/mm/Kconfig index 4788d59e5d36..d1135de34208 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -772,10 +772,18 @@ config DEFERRED_STRUCT_PAGE_INIT lifetime of the system until these kthreads finish the initialisation. +config PAGE_IDLE_FLAG + bool + select PAGE_EXTENSION if !64BIT + help + This adds PG_idle and PG_young flags to 'struct page'. PTE Accessed + bit writers can set the state of the bit in the flags so that PTE + Accessed bit readers may avoid disturbance. + config IDLE_PAGE_TRACKING bool "Enable idle page tracking" depends on SYSFS && MMU - select PAGE_EXTENSION if !64BIT + select PAGE_IDLE_FLAG help This feature allows to estimate the amount of user pages that have not been touched during a given period of time. This information can @@ -904,4 +912,6 @@ config ARCH_HAS_HUGEPD config MAPPING_DIRTY_HELPERS bool +source "mm/damon/Kconfig" + endmenu diff --git a/mm/Makefile b/mm/Makefile index d072cadb4299..a4afdc213150 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -117,6 +117,7 @@ obj-$(CONFIG_USERFAULTFD) += userfaultfd.o obj-$(CONFIG_IDLE_PAGE_TRACKING) += page_idle.o obj-$(CONFIG_FRAME_VECTOR) += frame_vector.o obj-$(CONFIG_DEBUG_PAGE_REF) += debug_page_ref.o +obj-$(CONFIG_DAMON) += damon/ obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o obj-$(CONFIG_ZONE_DEVICE) += memremap.o diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig new file mode 100644 index 000000000000..5bcf05851ad0 --- /dev/null +++ b/mm/damon/Kconfig @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: GPL-2.0-only + +menu "Data Access Monitoring" + +config DAMON + bool "DAMON: Data Access Monitoring Framework" + help + This builds a framework that allows kernel subsystems to monitor + access frequency of each memory region. The information can be useful + for performance-centric DRAM level memory management. + + See https://damonitor.github.io/doc/html/latest-damon/index.html for + more information. + +config DAMON_KUNIT_TEST + bool "Test for damon" if !KUNIT_ALL_TESTS + depends on DAMON && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the DAMON Kunit test suite. + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation. + + If unsure, say N. + +config DAMON_VADDR + bool "Data access monitoring primitives for virtual address spaces" + depends on DAMON && MMU + select PAGE_IDLE_FLAG + help + This builds the default data access monitoring primitives for DAMON + that work for virtual address spaces. + +config DAMON_PADDR + bool "Data access monitoring primitives for the physical address space" + depends on DAMON && MMU + select PAGE_IDLE_FLAG + help + This builds the default data access monitoring primitives for DAMON + that works for the physical address space. + +config DAMON_VADDR_KUNIT_TEST + bool "Test for DAMON primitives" if !KUNIT_ALL_TESTS + depends on DAMON_VADDR && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the DAMON virtual addresses primitives Kunit test suite. + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation. + + If unsure, say N. + +config DAMON_DBGFS + bool "DAMON debugfs interface" + depends on DAMON_VADDR && DAMON_PADDR && DEBUG_FS + help + This builds the debugfs interface for DAMON. The user space admins + can use the interface for arbitrary data access monitoring. + + If unsure, say N. + +config DAMON_DBGFS_KUNIT_TEST + bool "Test for damon debugfs interface" if !KUNIT_ALL_TESTS + depends on DAMON_DBGFS && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the DAMON debugfs interface Kunit test suite. + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation. + + If unsure, say N. + +config DAMON_RECLAIM + bool "Build DAMON-based reclaim (DAMON_RECLAIM)" + depends on DAMON_PADDR + help + This builds the DAMON-based reclamation subsystem. It finds pages + that not accessed for a long time (cold) using DAMON and reclaim + those. + + This is suggested to be used as a proactive and lightweight + reclamation under light memory pressure, while the traditional page + scanning-based reclamation is used for heavy pressure. + +endmenu diff --git a/mm/damon/Makefile b/mm/damon/Makefile new file mode 100644 index 000000000000..f7d5ac377a2b --- /dev/null +++ b/mm/damon/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DAMON) := core.o +obj-$(CONFIG_DAMON_VADDR) += prmtv-common.o vaddr.o +obj-$(CONFIG_DAMON_PADDR) += prmtv-common.o paddr.o +obj-$(CONFIG_DAMON_DBGFS) += dbgfs.o +obj-$(CONFIG_DAMON_RECLAIM) += reclaim.o diff --git a/mm/damon/core-test.h b/mm/damon/core-test.h new file mode 100644 index 000000000000..7008c3735e99 --- /dev/null +++ b/mm/damon/core-test.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Data Access Monitor Unit Tests + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Author: SeongJae Park + */ + +#ifdef CONFIG_DAMON_KUNIT_TEST + +#ifndef _DAMON_CORE_TEST_H +#define _DAMON_CORE_TEST_H + +#include + +static void damon_test_regions(struct kunit *test) +{ + struct damon_region *r; + struct damon_target *t; + + r = damon_new_region(1, 2); + KUNIT_EXPECT_EQ(test, 1ul, r->ar.start); + KUNIT_EXPECT_EQ(test, 2ul, r->ar.end); + KUNIT_EXPECT_EQ(test, 0u, r->nr_accesses); + + t = damon_new_target(42); + KUNIT_EXPECT_EQ(test, 0u, damon_nr_regions(t)); + + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, 1u, damon_nr_regions(t)); + + damon_del_region(r, t); + KUNIT_EXPECT_EQ(test, 0u, damon_nr_regions(t)); + + damon_free_target(t); +} + +static unsigned int nr_damon_targets(struct damon_ctx *ctx) +{ + struct damon_target *t; + unsigned int nr_targets = 0; + + damon_for_each_target(t, ctx) + nr_targets++; + + return nr_targets; +} + +static void damon_test_target(struct kunit *test) +{ + struct damon_ctx *c = damon_new_ctx(); + struct damon_target *t; + + t = damon_new_target(42); + KUNIT_EXPECT_EQ(test, 42ul, t->id); + KUNIT_EXPECT_EQ(test, 0u, nr_damon_targets(c)); + + damon_add_target(c, t); + KUNIT_EXPECT_EQ(test, 1u, nr_damon_targets(c)); + + damon_destroy_target(t); + KUNIT_EXPECT_EQ(test, 0u, nr_damon_targets(c)); + + damon_destroy_ctx(c); +} + +/* + * Test kdamond_reset_aggregated() + * + * DAMON checks access to each region and aggregates this information as the + * access frequency of each region. In detail, it increases '->nr_accesses' of + * regions that an access has confirmed. 'kdamond_reset_aggregated()' flushes + * the aggregated information ('->nr_accesses' of each regions) to the result + * buffer. As a result of the flushing, the '->nr_accesses' of regions are + * initialized to zero. + */ +static void damon_test_aggregate(struct kunit *test) +{ + struct damon_ctx *ctx = damon_new_ctx(); + unsigned long target_ids[] = {1, 2, 3}; + unsigned long saddr[][3] = {{10, 20, 30}, {5, 42, 49}, {13, 33, 55} }; + unsigned long eaddr[][3] = {{15, 27, 40}, {31, 45, 55}, {23, 44, 66} }; + unsigned long accesses[][3] = {{42, 95, 84}, {10, 20, 30}, {0, 1, 2} }; + struct damon_target *t; + struct damon_region *r; + int it, ir; + + damon_set_targets(ctx, target_ids, 3); + + it = 0; + damon_for_each_target(t, ctx) { + for (ir = 0; ir < 3; ir++) { + r = damon_new_region(saddr[it][ir], eaddr[it][ir]); + r->nr_accesses = accesses[it][ir]; + damon_add_region(r, t); + } + it++; + } + kdamond_reset_aggregated(ctx); + it = 0; + damon_for_each_target(t, ctx) { + ir = 0; + /* '->nr_accesses' should be zeroed */ + damon_for_each_region(r, t) { + KUNIT_EXPECT_EQ(test, 0u, r->nr_accesses); + ir++; + } + /* regions should be preserved */ + KUNIT_EXPECT_EQ(test, 3, ir); + it++; + } + /* targets also should be preserved */ + KUNIT_EXPECT_EQ(test, 3, it); + + damon_destroy_ctx(ctx); +} + +static void damon_test_split_at(struct kunit *test) +{ + struct damon_ctx *c = damon_new_ctx(); + struct damon_target *t; + struct damon_region *r; + + t = damon_new_target(42); + r = damon_new_region(0, 100); + damon_add_region(r, t); + damon_split_region_at(c, t, r, 25); + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); + KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); + + r = damon_next_region(r); + KUNIT_EXPECT_EQ(test, r->ar.start, 25ul); + KUNIT_EXPECT_EQ(test, r->ar.end, 100ul); + + damon_free_target(t); + damon_destroy_ctx(c); +} + +static void damon_test_merge_two(struct kunit *test) +{ + struct damon_target *t; + struct damon_region *r, *r2, *r3; + int i; + + t = damon_new_target(42); + r = damon_new_region(0, 100); + r->nr_accesses = 10; + damon_add_region(r, t); + r2 = damon_new_region(100, 300); + r2->nr_accesses = 20; + damon_add_region(r2, t); + + damon_merge_two_regions(t, r, r2); + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); + KUNIT_EXPECT_EQ(test, r->ar.end, 300ul); + KUNIT_EXPECT_EQ(test, r->nr_accesses, 16u); + + i = 0; + damon_for_each_region(r3, t) { + KUNIT_EXPECT_PTR_EQ(test, r, r3); + i++; + } + KUNIT_EXPECT_EQ(test, i, 1); + + damon_free_target(t); +} + +static struct damon_region *__nth_region_of(struct damon_target *t, int idx) +{ + struct damon_region *r; + unsigned int i = 0; + + damon_for_each_region(r, t) { + if (i++ == idx) + return r; + } + + return NULL; +} + +static void damon_test_merge_regions_of(struct kunit *test) +{ + struct damon_target *t; + struct damon_region *r; + unsigned long sa[] = {0, 100, 114, 122, 130, 156, 170, 184}; + unsigned long ea[] = {100, 112, 122, 130, 156, 170, 184, 230}; + unsigned int nrs[] = {0, 0, 10, 10, 20, 30, 1, 2}; + + unsigned long saddrs[] = {0, 114, 130, 156, 170}; + unsigned long eaddrs[] = {112, 130, 156, 170, 230}; + int i; + + t = damon_new_target(42); + for (i = 0; i < ARRAY_SIZE(sa); i++) { + r = damon_new_region(sa[i], ea[i]); + r->nr_accesses = nrs[i]; + damon_add_region(r, t); + } + + damon_merge_regions_of(t, 9, 9999); + /* 0-112, 114-130, 130-156, 156-170 */ + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u); + for (i = 0; i < 5; i++) { + r = __nth_region_of(t, i); + KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); + KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); + } + damon_free_target(t); +} + +static void damon_test_split_regions_of(struct kunit *test) +{ + struct damon_ctx *c = damon_new_ctx(); + struct damon_target *t; + struct damon_region *r; + + t = damon_new_target(42); + r = damon_new_region(0, 22); + damon_add_region(r, t); + damon_split_regions_of(c, t, 2); + KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u); + damon_free_target(t); + + t = damon_new_target(42); + r = damon_new_region(0, 220); + damon_add_region(r, t); + damon_split_regions_of(c, t, 4); + KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u); + damon_free_target(t); + damon_destroy_ctx(c); +} + +static struct kunit_case damon_test_cases[] = { + KUNIT_CASE(damon_test_target), + KUNIT_CASE(damon_test_regions), + KUNIT_CASE(damon_test_aggregate), + KUNIT_CASE(damon_test_split_at), + KUNIT_CASE(damon_test_merge_two), + KUNIT_CASE(damon_test_merge_regions_of), + KUNIT_CASE(damon_test_split_regions_of), + {}, +}; + +static struct kunit_suite damon_test_suite = { + .name = "damon", + .test_cases = damon_test_cases, +}; +kunit_test_suite(damon_test_suite); + +#endif /* _DAMON_CORE_TEST_H */ + +#endif /* CONFIG_DAMON_KUNIT_TEST */ diff --git a/mm/damon/core.c b/mm/damon/core.c new file mode 100644 index 000000000000..1dd153c31c9e --- /dev/null +++ b/mm/damon/core.c @@ -0,0 +1,1075 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Data Access Monitor + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon: " fmt + +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#ifdef CONFIG_DAMON_KUNIT_TEST +#undef DAMON_MIN_REGION +#define DAMON_MIN_REGION 1 +#endif + +static DEFINE_MUTEX(damon_lock); +static int nr_running_ctxs; + +/* + * Construct a damon_region struct + * + * Returns the pointer to the new struct if success, or NULL otherwise + */ +struct damon_region *damon_new_region(unsigned long start, unsigned long end) +{ + struct damon_region *region; + + region = kmalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return NULL; + + region->ar.start = start; + region->ar.end = end; + region->nr_accesses = 0; + INIT_LIST_HEAD(®ion->list); + + region->age = 0; + region->last_nr_accesses = 0; + + return region; +} + +void damon_add_region(struct damon_region *r, struct damon_target *t) +{ + list_add_tail(&r->list, &t->regions_list); + t->nr_regions++; +} + +static void damon_del_region(struct damon_region *r, struct damon_target *t) +{ + list_del(&r->list); + t->nr_regions--; +} + +static void damon_free_region(struct damon_region *r) +{ + kfree(r); +} + +void damon_destroy_region(struct damon_region *r, struct damon_target *t) +{ + damon_del_region(r, t); + damon_free_region(r); +} + +struct damos *damon_new_scheme( + unsigned long min_sz_region, unsigned long max_sz_region, + unsigned int min_nr_accesses, unsigned int max_nr_accesses, + unsigned int min_age_region, unsigned int max_age_region, + enum damos_action action, struct damos_quota *quota, + struct damos_watermarks *wmarks) +{ + struct damos *scheme; + + scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); + if (!scheme) + return NULL; + scheme->min_sz_region = min_sz_region; + scheme->max_sz_region = max_sz_region; + scheme->min_nr_accesses = min_nr_accesses; + scheme->max_nr_accesses = max_nr_accesses; + scheme->min_age_region = min_age_region; + scheme->max_age_region = max_age_region; + scheme->action = action; + scheme->stat = (struct damos_stat){}; + INIT_LIST_HEAD(&scheme->list); + + scheme->quota.ms = quota->ms; + scheme->quota.sz = quota->sz; + scheme->quota.reset_interval = quota->reset_interval; + scheme->quota.weight_sz = quota->weight_sz; + scheme->quota.weight_nr_accesses = quota->weight_nr_accesses; + scheme->quota.weight_age = quota->weight_age; + scheme->quota.total_charged_sz = 0; + scheme->quota.total_charged_ns = 0; + scheme->quota.esz = 0; + scheme->quota.charged_sz = 0; + scheme->quota.charged_from = 0; + scheme->quota.charge_target_from = NULL; + scheme->quota.charge_addr_from = 0; + + scheme->wmarks.metric = wmarks->metric; + scheme->wmarks.interval = wmarks->interval; + scheme->wmarks.high = wmarks->high; + scheme->wmarks.mid = wmarks->mid; + scheme->wmarks.low = wmarks->low; + scheme->wmarks.activated = true; + + return scheme; +} + +void damon_add_scheme(struct damon_ctx *ctx, struct damos *s) +{ + list_add_tail(&s->list, &ctx->schemes); +} + +static void damon_del_scheme(struct damos *s) +{ + list_del(&s->list); +} + +static void damon_free_scheme(struct damos *s) +{ + kfree(s); +} + +void damon_destroy_scheme(struct damos *s) +{ + damon_del_scheme(s); + damon_free_scheme(s); +} + +/* + * Construct a damon_target struct + * + * Returns the pointer to the new struct if success, or NULL otherwise + */ +struct damon_target *damon_new_target(unsigned long id) +{ + struct damon_target *t; + + t = kmalloc(sizeof(*t), GFP_KERNEL); + if (!t) + return NULL; + + t->id = id; + t->nr_regions = 0; + INIT_LIST_HEAD(&t->regions_list); + + return t; +} + +void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) +{ + list_add_tail(&t->list, &ctx->adaptive_targets); +} + +bool damon_targets_empty(struct damon_ctx *ctx) +{ + return list_empty(&ctx->adaptive_targets); +} + +static void damon_del_target(struct damon_target *t) +{ + list_del(&t->list); +} + +void damon_free_target(struct damon_target *t) +{ + struct damon_region *r, *next; + + damon_for_each_region_safe(r, next, t) + damon_free_region(r); + kfree(t); +} + +void damon_destroy_target(struct damon_target *t) +{ + damon_del_target(t); + damon_free_target(t); +} + +unsigned int damon_nr_regions(struct damon_target *t) +{ + return t->nr_regions; +} + +struct damon_ctx *damon_new_ctx(void) +{ + struct damon_ctx *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->sample_interval = 5 * 1000; + ctx->aggr_interval = 100 * 1000; + ctx->primitive_update_interval = 60 * 1000 * 1000; + + ktime_get_coarse_ts64(&ctx->last_aggregation); + ctx->last_primitive_update = ctx->last_aggregation; + + mutex_init(&ctx->kdamond_lock); + + ctx->min_nr_regions = 10; + ctx->max_nr_regions = 1000; + + INIT_LIST_HEAD(&ctx->adaptive_targets); + INIT_LIST_HEAD(&ctx->schemes); + + return ctx; +} + +static void damon_destroy_targets(struct damon_ctx *ctx) +{ + struct damon_target *t, *next_t; + + if (ctx->primitive.cleanup) { + ctx->primitive.cleanup(ctx); + return; + } + + damon_for_each_target_safe(t, next_t, ctx) + damon_destroy_target(t); +} + +void damon_destroy_ctx(struct damon_ctx *ctx) +{ + struct damos *s, *next_s; + + damon_destroy_targets(ctx); + + damon_for_each_scheme_safe(s, next_s, ctx) + damon_destroy_scheme(s); + + kfree(ctx); +} + +/** + * damon_set_targets() - Set monitoring targets. + * @ctx: monitoring context + * @ids: array of target ids + * @nr_ids: number of entries in @ids + * + * This function should not be called while the kdamond is running. + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_set_targets(struct damon_ctx *ctx, + unsigned long *ids, ssize_t nr_ids) +{ + ssize_t i; + struct damon_target *t, *next; + + damon_destroy_targets(ctx); + + for (i = 0; i < nr_ids; i++) { + t = damon_new_target(ids[i]); + if (!t) { + /* The caller should do cleanup of the ids itself */ + damon_for_each_target_safe(t, next, ctx) + damon_destroy_target(t); + return -ENOMEM; + } + damon_add_target(ctx, t); + } + + return 0; +} + +/** + * damon_set_attrs() - Set attributes for the monitoring. + * @ctx: monitoring context + * @sample_int: time interval between samplings + * @aggr_int: time interval between aggregations + * @primitive_upd_int: time interval between monitoring primitive updates + * @min_nr_reg: minimal number of regions + * @max_nr_reg: maximum number of regions + * + * This function should not be called while the kdamond is running. + * Every time interval is in micro-seconds. + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, + unsigned long aggr_int, unsigned long primitive_upd_int, + unsigned long min_nr_reg, unsigned long max_nr_reg) +{ + if (min_nr_reg < 3) + return -EINVAL; + if (min_nr_reg > max_nr_reg) + return -EINVAL; + + ctx->sample_interval = sample_int; + ctx->aggr_interval = aggr_int; + ctx->primitive_update_interval = primitive_upd_int; + ctx->min_nr_regions = min_nr_reg; + ctx->max_nr_regions = max_nr_reg; + + return 0; +} + +/** + * damon_set_schemes() - Set data access monitoring based operation schemes. + * @ctx: monitoring context + * @schemes: array of the schemes + * @nr_schemes: number of entries in @schemes + * + * This function should not be called while the kdamond of the context is + * running. + * + * Return: 0 if success, or negative error code otherwise. + */ +int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, + ssize_t nr_schemes) +{ + struct damos *s, *next; + ssize_t i; + + damon_for_each_scheme_safe(s, next, ctx) + damon_destroy_scheme(s); + for (i = 0; i < nr_schemes; i++) + damon_add_scheme(ctx, schemes[i]); + return 0; +} + +/** + * damon_nr_running_ctxs() - Return number of currently running contexts. + */ +int damon_nr_running_ctxs(void) +{ + int nr_ctxs; + + mutex_lock(&damon_lock); + nr_ctxs = nr_running_ctxs; + mutex_unlock(&damon_lock); + + return nr_ctxs; +} + +/* Returns the size upper limit for each monitoring region */ +static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct damon_region *r; + unsigned long sz = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) + sz += r->ar.end - r->ar.start; + } + + if (ctx->min_nr_regions) + sz /= ctx->min_nr_regions; + if (sz < DAMON_MIN_REGION) + sz = DAMON_MIN_REGION; + + return sz; +} + +static int kdamond_fn(void *data); + +/* + * __damon_start() - Starts monitoring with given context. + * @ctx: monitoring context + * + * This function should be called while damon_lock is hold. + * + * Return: 0 on success, negative error code otherwise. + */ +static int __damon_start(struct damon_ctx *ctx) +{ + int err = -EBUSY; + + mutex_lock(&ctx->kdamond_lock); + if (!ctx->kdamond) { + err = 0; + ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", + nr_running_ctxs); + if (IS_ERR(ctx->kdamond)) { + err = PTR_ERR(ctx->kdamond); + ctx->kdamond = NULL; + } + } + mutex_unlock(&ctx->kdamond_lock); + + return err; +} + +/** + * damon_start() - Starts the monitorings for a given group of contexts. + * @ctxs: an array of the pointers for contexts to start monitoring + * @nr_ctxs: size of @ctxs + * + * This function starts a group of monitoring threads for a group of monitoring + * contexts. One thread per each context is created and run in parallel. The + * caller should handle synchronization between the threads by itself. If a + * group of threads that created by other 'damon_start()' call is currently + * running, this function does nothing but returns -EBUSY. + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_start(struct damon_ctx **ctxs, int nr_ctxs) +{ + int i; + int err = 0; + + mutex_lock(&damon_lock); + if (nr_running_ctxs) { + mutex_unlock(&damon_lock); + return -EBUSY; + } + + for (i = 0; i < nr_ctxs; i++) { + err = __damon_start(ctxs[i]); + if (err) + break; + nr_running_ctxs++; + } + mutex_unlock(&damon_lock); + + return err; +} + +/* + * __damon_stop() - Stops monitoring of given context. + * @ctx: monitoring context + * + * Return: 0 on success, negative error code otherwise. + */ +static int __damon_stop(struct damon_ctx *ctx) +{ + struct task_struct *tsk; + + mutex_lock(&ctx->kdamond_lock); + tsk = ctx->kdamond; + if (tsk) { + get_task_struct(tsk); + mutex_unlock(&ctx->kdamond_lock); + kthread_stop(tsk); + put_task_struct(tsk); + return 0; + } + mutex_unlock(&ctx->kdamond_lock); + + return -EPERM; +} + +/** + * damon_stop() - Stops the monitorings for a given group of contexts. + * @ctxs: an array of the pointers for contexts to stop monitoring + * @nr_ctxs: size of @ctxs + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) +{ + int i, err = 0; + + for (i = 0; i < nr_ctxs; i++) { + /* nr_running_ctxs is decremented in kdamond_fn */ + err = __damon_stop(ctxs[i]); + if (err) + return err; + } + + return err; +} + +/* + * damon_check_reset_time_interval() - Check if a time interval is elapsed. + * @baseline: the time to check whether the interval has elapsed since + * @interval: the time interval (microseconds) + * + * See whether the given time interval has passed since the given baseline + * time. If so, it also updates the baseline to current time for next check. + * + * Return: true if the time interval has passed, or false otherwise. + */ +static bool damon_check_reset_time_interval(struct timespec64 *baseline, + unsigned long interval) +{ + struct timespec64 now; + + ktime_get_coarse_ts64(&now); + if ((timespec64_to_ns(&now) - timespec64_to_ns(baseline)) < + interval * 1000) + return false; + *baseline = now; + return true; +} + +/* + * Check whether it is time to flush the aggregated information + */ +static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) +{ + return damon_check_reset_time_interval(&ctx->last_aggregation, + ctx->aggr_interval); +} + +/* + * Reset the aggregated monitoring results ('nr_accesses' of each region). + */ +static void kdamond_reset_aggregated(struct damon_ctx *c) +{ + struct damon_target *t; + unsigned int ti = 0; /* target's index */ + + damon_for_each_target(t, c) { + struct damon_region *r; + + damon_for_each_region(r, t) { + trace_damon_aggregated(t, ti, r, damon_nr_regions(t)); + r->last_nr_accesses = r->nr_accesses; + r->nr_accesses = 0; + } + ti++; + } +} + +static void damon_split_region_at(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + unsigned long sz_r); + +static bool __damos_valid_target(struct damon_region *r, struct damos *s) +{ + unsigned long sz; + + sz = r->ar.end - r->ar.start; + return s->min_sz_region <= sz && sz <= s->max_sz_region && + s->min_nr_accesses <= r->nr_accesses && + r->nr_accesses <= s->max_nr_accesses && + s->min_age_region <= r->age && r->age <= s->max_age_region; +} + +static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, + struct damon_region *r, struct damos *s) +{ + bool ret = __damos_valid_target(r, s); + + if (!ret || !s->quota.esz || !c->primitive.get_scheme_score) + return ret; + + return c->primitive.get_scheme_score(c, t, r, s) >= s->quota.min_score; +} + +static void damon_do_apply_schemes(struct damon_ctx *c, + struct damon_target *t, + struct damon_region *r) +{ + struct damos *s; + + damon_for_each_scheme(s, c) { + struct damos_quota *quota = &s->quota; + unsigned long sz = r->ar.end - r->ar.start; + struct timespec64 begin, end; + unsigned long sz_applied = 0; + + if (!s->wmarks.activated) + continue; + + /* Check the quota */ + if (quota->esz && quota->charged_sz >= quota->esz) + continue; + + /* Skip previously charged regions */ + if (quota->charge_target_from) { + if (t != quota->charge_target_from) + continue; + if (r == damon_last_region(t)) { + quota->charge_target_from = NULL; + quota->charge_addr_from = 0; + continue; + } + if (quota->charge_addr_from && + r->ar.end <= quota->charge_addr_from) + continue; + + if (quota->charge_addr_from && r->ar.start < + quota->charge_addr_from) { + sz = ALIGN_DOWN(quota->charge_addr_from - + r->ar.start, DAMON_MIN_REGION); + if (!sz) { + if (r->ar.end - r->ar.start <= + DAMON_MIN_REGION) + continue; + sz = DAMON_MIN_REGION; + } + damon_split_region_at(c, t, r, sz); + r = damon_next_region(r); + sz = r->ar.end - r->ar.start; + } + quota->charge_target_from = NULL; + quota->charge_addr_from = 0; + } + + if (!damos_valid_target(c, t, r, s)) + continue; + + /* Apply the scheme */ + if (c->primitive.apply_scheme) { + if (quota->esz && + quota->charged_sz + sz > quota->esz) { + sz = ALIGN_DOWN(quota->esz - quota->charged_sz, + DAMON_MIN_REGION); + if (!sz) + goto update_stat; + damon_split_region_at(c, t, r, sz); + } + ktime_get_coarse_ts64(&begin); + sz_applied = c->primitive.apply_scheme(c, t, r, s); + ktime_get_coarse_ts64(&end); + quota->total_charged_ns += timespec64_to_ns(&end) - + timespec64_to_ns(&begin); + quota->charged_sz += sz; + if (quota->esz && quota->charged_sz >= quota->esz) { + quota->charge_target_from = t; + quota->charge_addr_from = r->ar.end + 1; + } + } + if (s->action != DAMOS_STAT) + r->age = 0; + +update_stat: + s->stat.nr_tried++; + s->stat.sz_tried += sz; + if (sz_applied) + s->stat.nr_applied++; + s->stat.sz_applied += sz_applied; + } +} + +/* Shouldn't be called if quota->ms and quota->sz are zero */ +static void damos_set_effective_quota(struct damos_quota *quota) +{ + unsigned long throughput; + unsigned long esz; + + if (!quota->ms) { + quota->esz = quota->sz; + return; + } + + if (quota->total_charged_ns) + throughput = quota->total_charged_sz * 1000000 / + quota->total_charged_ns; + else + throughput = PAGE_SIZE * 1024; + esz = throughput * quota->ms; + + if (quota->sz && quota->sz < esz) + esz = quota->sz; + quota->esz = esz; +} + +static void kdamond_apply_schemes(struct damon_ctx *c) +{ + struct damon_target *t; + struct damon_region *r, *next_r; + struct damos *s; + + damon_for_each_scheme(s, c) { + struct damos_quota *quota = &s->quota; + unsigned long cumulated_sz; + unsigned int score, max_score = 0; + + if (!s->wmarks.activated) + continue; + + if (!quota->ms && !quota->sz) + continue; + + /* New charge window starts */ + if (time_after_eq(jiffies, quota->charged_from + + msecs_to_jiffies( + quota->reset_interval))) { + if (quota->esz && quota->charged_sz >= quota->esz) + s->stat.qt_exceeds++; + quota->total_charged_sz += quota->charged_sz; + quota->charged_from = jiffies; + quota->charged_sz = 0; + damos_set_effective_quota(quota); + } + + if (!c->primitive.get_scheme_score) + continue; + + /* Fill up the score histogram */ + memset(quota->histogram, 0, sizeof(quota->histogram)); + damon_for_each_target(t, c) { + damon_for_each_region(r, t) { + if (!__damos_valid_target(r, s)) + continue; + score = c->primitive.get_scheme_score( + c, t, r, s); + quota->histogram[score] += + r->ar.end - r->ar.start; + if (score > max_score) + max_score = score; + } + } + + /* Set the min score limit */ + for (cumulated_sz = 0, score = max_score; ; score--) { + cumulated_sz += quota->histogram[score]; + if (cumulated_sz >= quota->esz || !score) + break; + } + quota->min_score = score; + } + + damon_for_each_target(t, c) { + damon_for_each_region_safe(r, next_r, t) + damon_do_apply_schemes(c, t, r); + } +} + +static inline unsigned long sz_damon_region(struct damon_region *r) +{ + return r->ar.end - r->ar.start; +} + +/* + * Merge two adjacent regions into one region + */ +static void damon_merge_two_regions(struct damon_target *t, + struct damon_region *l, struct damon_region *r) +{ + unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r); + + l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / + (sz_l + sz_r); + l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); + l->ar.end = r->ar.end; + damon_destroy_region(r, t); +} + +/* + * Merge adjacent regions having similar access frequencies + * + * t target affected by this merge operation + * thres '->nr_accesses' diff threshold for the merge + * sz_limit size upper limit of each region + */ +static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, + unsigned long sz_limit) +{ + struct damon_region *r, *prev = NULL, *next; + + damon_for_each_region_safe(r, next, t) { + if (abs(r->nr_accesses - r->last_nr_accesses) > thres) + r->age = 0; + else + r->age++; + + if (prev && prev->ar.end == r->ar.start && + abs(prev->nr_accesses - r->nr_accesses) <= thres && + sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) + damon_merge_two_regions(t, prev, r); + else + prev = r; + } +} + +/* + * Merge adjacent regions having similar access frequencies + * + * threshold '->nr_accesses' diff threshold for the merge + * sz_limit size upper limit of each region + * + * This function merges monitoring target regions which are adjacent and their + * access frequencies are similar. This is for minimizing the monitoring + * overhead under the dynamically changeable access pattern. If a merge was + * unnecessarily made, later 'kdamond_split_regions()' will revert it. + */ +static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, + unsigned long sz_limit) +{ + struct damon_target *t; + + damon_for_each_target(t, c) + damon_merge_regions_of(t, threshold, sz_limit); +} + +/* + * Split a region in two + * + * r the region to be split + * sz_r size of the first sub-region that will be made + */ +static void damon_split_region_at(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + unsigned long sz_r) +{ + struct damon_region *new; + + new = damon_new_region(r->ar.start + sz_r, r->ar.end); + if (!new) + return; + + r->ar.end = new->ar.start; + + new->age = r->age; + new->last_nr_accesses = r->last_nr_accesses; + + damon_insert_region(new, r, damon_next_region(r), t); +} + +/* Split every region in the given target into 'nr_subs' regions */ +static void damon_split_regions_of(struct damon_ctx *ctx, + struct damon_target *t, int nr_subs) +{ + struct damon_region *r, *next; + unsigned long sz_region, sz_sub = 0; + int i; + + damon_for_each_region_safe(r, next, t) { + sz_region = r->ar.end - r->ar.start; + + for (i = 0; i < nr_subs - 1 && + sz_region > 2 * DAMON_MIN_REGION; i++) { + /* + * Randomly select size of left sub-region to be at + * least 10 percent and at most 90% of original region + */ + sz_sub = ALIGN_DOWN(damon_rand(1, 10) * + sz_region / 10, DAMON_MIN_REGION); + /* Do not allow blank region */ + if (sz_sub == 0 || sz_sub >= sz_region) + continue; + + damon_split_region_at(ctx, t, r, sz_sub); + sz_region = sz_sub; + } + } +} + +/* + * Split every target region into randomly-sized small regions + * + * This function splits every target region into random-sized small regions if + * current total number of the regions is equal or smaller than half of the + * user-specified maximum number of regions. This is for maximizing the + * monitoring accuracy under the dynamically changeable access patterns. If a + * split was unnecessarily made, later 'kdamond_merge_regions()' will revert + * it. + */ +static void kdamond_split_regions(struct damon_ctx *ctx) +{ + struct damon_target *t; + unsigned int nr_regions = 0; + static unsigned int last_nr_regions; + int nr_subregions = 2; + + damon_for_each_target(t, ctx) + nr_regions += damon_nr_regions(t); + + if (nr_regions > ctx->max_nr_regions / 2) + return; + + /* Maybe the middle of the region has different access frequency */ + if (last_nr_regions == nr_regions && + nr_regions < ctx->max_nr_regions / 3) + nr_subregions = 3; + + damon_for_each_target(t, ctx) + damon_split_regions_of(ctx, t, nr_subregions); + + last_nr_regions = nr_regions; +} + +/* + * Check whether it is time to check and apply the target monitoring regions + * + * Returns true if it is. + */ +static bool kdamond_need_update_primitive(struct damon_ctx *ctx) +{ + return damon_check_reset_time_interval(&ctx->last_primitive_update, + ctx->primitive_update_interval); +} + +/* + * Check whether current monitoring should be stopped + * + * The monitoring is stopped when either the user requested to stop, or all + * monitoring targets are invalid. + * + * Returns true if need to stop current monitoring. + */ +static bool kdamond_need_stop(struct damon_ctx *ctx) +{ + struct damon_target *t; + + if (kthread_should_stop()) + return true; + + if (!ctx->primitive.target_valid) + return false; + + damon_for_each_target(t, ctx) { + if (ctx->primitive.target_valid(t)) + return false; + } + + return true; +} + +static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) +{ + struct sysinfo i; + + switch (metric) { + case DAMOS_WMARK_FREE_MEM_RATE: + si_meminfo(&i); + return i.freeram * 1000 / i.totalram; + default: + break; + } + return -EINVAL; +} + +/* + * Returns zero if the scheme is active. Else, returns time to wait for next + * watermark check in micro-seconds. + */ +static unsigned long damos_wmark_wait_us(struct damos *scheme) +{ + unsigned long metric; + + if (scheme->wmarks.metric == DAMOS_WMARK_NONE) + return 0; + + metric = damos_wmark_metric_value(scheme->wmarks.metric); + /* higher than high watermark or lower than low watermark */ + if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { + if (scheme->wmarks.activated) + pr_debug("deactivate a scheme (%d) for %s wmark\n", + scheme->action, + metric > scheme->wmarks.high ? + "high" : "low"); + scheme->wmarks.activated = false; + return scheme->wmarks.interval; + } + + /* inactive and higher than middle watermark */ + if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) && + !scheme->wmarks.activated) + return scheme->wmarks.interval; + + if (!scheme->wmarks.activated) + pr_debug("activate a scheme (%d)\n", scheme->action); + scheme->wmarks.activated = true; + return 0; +} + +static void kdamond_usleep(unsigned long usecs) +{ + /* See Documentation/timers/timers-howto.rst for the thresholds */ + if (usecs > 20 * USEC_PER_MSEC) + schedule_timeout_idle(usecs_to_jiffies(usecs)); + else + usleep_idle_range(usecs, usecs + 1); +} + +/* Returns negative error code if it's not activated but should return */ +static int kdamond_wait_activation(struct damon_ctx *ctx) +{ + struct damos *s; + unsigned long wait_time; + unsigned long min_wait_time = 0; + + while (!kdamond_need_stop(ctx)) { + damon_for_each_scheme(s, ctx) { + wait_time = damos_wmark_wait_us(s); + if (!min_wait_time || wait_time < min_wait_time) + min_wait_time = wait_time; + } + if (!min_wait_time) + return 0; + + kdamond_usleep(min_wait_time); + } + return -EBUSY; +} + +/* + * The monitoring daemon that runs as a kernel thread + */ +static int kdamond_fn(void *data) +{ + struct damon_ctx *ctx = (struct damon_ctx *)data; + struct damon_target *t; + struct damon_region *r, *next; + unsigned int max_nr_accesses = 0; + unsigned long sz_limit = 0; + bool done = false; + + pr_debug("kdamond (%d) starts\n", current->pid); + + if (ctx->primitive.init) + ctx->primitive.init(ctx); + if (ctx->callback.before_start && ctx->callback.before_start(ctx)) + done = true; + + sz_limit = damon_region_sz_limit(ctx); + + while (!kdamond_need_stop(ctx) && !done) { + if (kdamond_wait_activation(ctx)) + continue; + + if (ctx->primitive.prepare_access_checks) + ctx->primitive.prepare_access_checks(ctx); + if (ctx->callback.after_sampling && + ctx->callback.after_sampling(ctx)) + done = true; + + kdamond_usleep(ctx->sample_interval); + + if (ctx->primitive.check_accesses) + max_nr_accesses = ctx->primitive.check_accesses(ctx); + + if (kdamond_aggregate_interval_passed(ctx)) { + kdamond_merge_regions(ctx, + max_nr_accesses / 10, + sz_limit); + if (ctx->callback.after_aggregation && + ctx->callback.after_aggregation(ctx)) + done = true; + kdamond_apply_schemes(ctx); + kdamond_reset_aggregated(ctx); + kdamond_split_regions(ctx); + if (ctx->primitive.reset_aggregated) + ctx->primitive.reset_aggregated(ctx); + } + + if (kdamond_need_update_primitive(ctx)) { + if (ctx->primitive.update) + ctx->primitive.update(ctx); + sz_limit = damon_region_sz_limit(ctx); + } + } + damon_for_each_target(t, ctx) { + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + } + + if (ctx->callback.before_terminate) + ctx->callback.before_terminate(ctx); + if (ctx->primitive.cleanup) + ctx->primitive.cleanup(ctx); + + pr_debug("kdamond (%d) finishes\n", current->pid); + mutex_lock(&ctx->kdamond_lock); + ctx->kdamond = NULL; + mutex_unlock(&ctx->kdamond_lock); + + mutex_lock(&damon_lock); + nr_running_ctxs--; + mutex_unlock(&damon_lock); + + return 0; +} + +#include "core-test.h" diff --git a/mm/damon/dbgfs-test.h b/mm/damon/dbgfs-test.h new file mode 100644 index 000000000000..86b9f9528231 --- /dev/null +++ b/mm/damon/dbgfs-test.h @@ -0,0 +1,180 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DAMON Debugfs Interface Unit Tests + * + * Author: SeongJae Park + */ + +#ifdef CONFIG_DAMON_DBGFS_KUNIT_TEST + +#ifndef _DAMON_DBGFS_TEST_H +#define _DAMON_DBGFS_TEST_H + +#include + +static void damon_dbgfs_test_str_to_target_ids(struct kunit *test) +{ + char *question; + unsigned long *answers; + unsigned long expected[] = {12, 35, 46}; + ssize_t nr_integers = 0, i; + + question = "123"; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers); + KUNIT_EXPECT_EQ(test, 123ul, answers[0]); + kfree(answers); + + question = "123abc"; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)1, nr_integers); + KUNIT_EXPECT_EQ(test, 123ul, answers[0]); + kfree(answers); + + question = "a123"; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); + kfree(answers); + + question = "12 35"; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers); + for (i = 0; i < nr_integers; i++) + KUNIT_EXPECT_EQ(test, expected[i], answers[i]); + kfree(answers); + + question = "12 35 46"; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)3, nr_integers); + for (i = 0; i < nr_integers; i++) + KUNIT_EXPECT_EQ(test, expected[i], answers[i]); + kfree(answers); + + question = "12 35 abc 46"; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)2, nr_integers); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, expected[i], answers[i]); + kfree(answers); + + question = ""; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); + kfree(answers); + + question = "\n"; + answers = str_to_target_ids(question, strlen(question), + &nr_integers); + KUNIT_EXPECT_EQ(test, (ssize_t)0, nr_integers); + kfree(answers); +} + +static void damon_dbgfs_test_set_targets(struct kunit *test) +{ + struct damon_ctx *ctx = dbgfs_new_ctx(); + unsigned long ids[] = {1, 2, 3}; + char buf[64]; + + /* Make DAMON consider target id as plain number */ + ctx->primitive.target_valid = NULL; + ctx->primitive.cleanup = NULL; + + damon_set_targets(ctx, ids, 3); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "1 2 3\n"); + + damon_set_targets(ctx, NULL, 0); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "\n"); + + damon_set_targets(ctx, (unsigned long []){1, 2}, 2); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "1 2\n"); + + damon_set_targets(ctx, (unsigned long []){2}, 1); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "2\n"); + + damon_set_targets(ctx, NULL, 0); + sprint_target_ids(ctx, buf, 64); + KUNIT_EXPECT_STREQ(test, (char *)buf, "\n"); + + dbgfs_destroy_ctx(ctx); +} + +static void damon_dbgfs_test_set_init_regions(struct kunit *test) +{ + struct damon_ctx *ctx = damon_new_ctx(); + unsigned long ids[] = {1, 2, 3}; + /* Each line represents one region in `` `` */ + char * const valid_inputs[] = {"2 10 20\n 2 20 30\n2 35 45", + "2 10 20\n", + "2 10 20\n1 39 59\n1 70 134\n 2 20 25\n", + ""}; + /* Reading the file again will show sorted, clean output */ + char * const valid_expects[] = {"2 10 20\n2 20 30\n2 35 45\n", + "2 10 20\n", + "1 39 59\n1 70 134\n2 10 20\n2 20 25\n", + ""}; + char * const invalid_inputs[] = {"4 10 20\n", /* target not exists */ + "2 10 20\n 2 14 26\n", /* regions overlap */ + "1 10 20\n2 30 40\n 1 5 8"}; /* not sorted by address */ + char *input, *expect; + int i, rc; + char buf[256]; + + damon_set_targets(ctx, ids, 3); + + /* Put valid inputs and check the results */ + for (i = 0; i < ARRAY_SIZE(valid_inputs); i++) { + input = valid_inputs[i]; + expect = valid_expects[i]; + + rc = set_init_regions(ctx, input, strnlen(input, 256)); + KUNIT_EXPECT_EQ(test, rc, 0); + + memset(buf, 0, 256); + sprint_init_regions(ctx, buf, 256); + + KUNIT_EXPECT_STREQ(test, (char *)buf, expect); + } + /* Put invalid inputs and check the return error code */ + for (i = 0; i < ARRAY_SIZE(invalid_inputs); i++) { + input = invalid_inputs[i]; + pr_info("input: %s\n", input); + rc = set_init_regions(ctx, input, strnlen(input, 256)); + KUNIT_EXPECT_EQ(test, rc, -EINVAL); + + memset(buf, 0, 256); + sprint_init_regions(ctx, buf, 256); + + KUNIT_EXPECT_STREQ(test, (char *)buf, ""); + } + + damon_set_targets(ctx, NULL, 0); + damon_destroy_ctx(ctx); +} + +static struct kunit_case damon_test_cases[] = { + KUNIT_CASE(damon_dbgfs_test_str_to_target_ids), + KUNIT_CASE(damon_dbgfs_test_set_targets), + KUNIT_CASE(damon_dbgfs_test_set_init_regions), + {}, +}; + +static struct kunit_suite damon_test_suite = { + .name = "damon-dbgfs", + .test_cases = damon_test_cases, +}; +kunit_test_suite(damon_test_suite); + +#endif /* _DAMON_TEST_H */ + +#endif /* CONFIG_DAMON_KUNIT_TEST */ diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c new file mode 100644 index 000000000000..5b899601e56c --- /dev/null +++ b/mm/damon/dbgfs.c @@ -0,0 +1,990 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON Debugfs Interface + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-dbgfs: " fmt + +#include +#include +#include +#include +#include +#include +#include + +static struct damon_ctx **dbgfs_ctxs; +static int dbgfs_nr_ctxs; +static struct dentry **dbgfs_dirs; +static DEFINE_MUTEX(damon_dbgfs_lock); + +/* + * Returns non-empty string on success, negative error code otherwise. + */ +static char *user_input_str(const char __user *buf, size_t count, loff_t *ppos) +{ + char *kbuf; + ssize_t ret; + + /* We do not accept continuous write */ + if (*ppos) + return ERR_PTR(-EINVAL); + + kbuf = kmalloc(count + 1, GFP_KERNEL | __GFP_NOWARN); + if (!kbuf) + return ERR_PTR(-ENOMEM); + + ret = simple_write_to_buffer(kbuf, count + 1, ppos, buf, count); + if (ret != count) { + kfree(kbuf); + return ERR_PTR(-EIO); + } + kbuf[ret] = '\0'; + + return kbuf; +} + +static ssize_t dbgfs_attrs_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char kbuf[128]; + int ret; + + mutex_lock(&ctx->kdamond_lock); + ret = scnprintf(kbuf, ARRAY_SIZE(kbuf), "%lu %lu %lu %lu %lu\n", + ctx->sample_interval, ctx->aggr_interval, + ctx->primitive_update_interval, ctx->min_nr_regions, + ctx->max_nr_regions); + mutex_unlock(&ctx->kdamond_lock); + + return simple_read_from_buffer(buf, count, ppos, kbuf, ret); +} + +static ssize_t dbgfs_attrs_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + unsigned long s, a, r, minr, maxr; + char *kbuf; + ssize_t ret; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + if (sscanf(kbuf, "%lu %lu %lu %lu %lu", + &s, &a, &r, &minr, &maxr) != 5) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + ret = -EBUSY; + goto unlock_out; + } + + ret = damon_set_attrs(ctx, s, a, r, minr, maxr); + if (!ret) + ret = count; +unlock_out: + mutex_unlock(&ctx->kdamond_lock); +out: + kfree(kbuf); + return ret; +} + +static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) +{ + struct damos *s; + int written = 0; + int rc; + + damon_for_each_scheme(s, c) { + rc = scnprintf(&buf[written], len - written, + "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu %lu %lu %lu\n", + s->min_sz_region, s->max_sz_region, + s->min_nr_accesses, s->max_nr_accesses, + s->min_age_region, s->max_age_region, + s->action, + s->quota.ms, s->quota.sz, + s->quota.reset_interval, + s->quota.weight_sz, + s->quota.weight_nr_accesses, + s->quota.weight_age, + s->wmarks.metric, s->wmarks.interval, + s->wmarks.high, s->wmarks.mid, s->wmarks.low, + s->stat.nr_tried, s->stat.sz_tried, + s->stat.nr_applied, s->stat.sz_applied, + s->stat.qt_exceeds); + if (!rc) + return -ENOMEM; + + written += rc; + } + return written; +} + +static ssize_t dbgfs_schemes_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t len; + + kbuf = kmalloc(count, GFP_KERNEL | __GFP_NOWARN); + if (!kbuf) + return -ENOMEM; + + mutex_lock(&ctx->kdamond_lock); + len = sprint_schemes(ctx, kbuf, count); + mutex_unlock(&ctx->kdamond_lock); + if (len < 0) + goto out; + len = simple_read_from_buffer(buf, count, ppos, kbuf, len); + +out: + kfree(kbuf); + return len; +} + +static void free_schemes_arr(struct damos **schemes, ssize_t nr_schemes) +{ + ssize_t i; + + for (i = 0; i < nr_schemes; i++) + kfree(schemes[i]); + kfree(schemes); +} + +static bool damos_action_valid(int action) +{ + switch (action) { + case DAMOS_WILLNEED: + case DAMOS_COLD: + case DAMOS_PAGEOUT: + case DAMOS_HUGEPAGE: + case DAMOS_NOHUGEPAGE: + case DAMOS_STAT: + return true; + default: + return false; + } +} + +/* + * Converts a string into an array of struct damos pointers + * + * Returns an array of struct damos pointers that converted if the conversion + * success, or NULL otherwise. + */ +static struct damos **str_to_schemes(const char *str, ssize_t len, + ssize_t *nr_schemes) +{ + struct damos *scheme, **schemes; + const int max_nr_schemes = 256; + int pos = 0, parsed, ret; + unsigned long min_sz, max_sz; + unsigned int min_nr_a, max_nr_a, min_age, max_age; + unsigned int action; + + schemes = kmalloc_array(max_nr_schemes, sizeof(scheme), + GFP_KERNEL); + if (!schemes) + return NULL; + + *nr_schemes = 0; + while (pos < len && *nr_schemes < max_nr_schemes) { + struct damos_quota quota = {}; + struct damos_watermarks wmarks; + + ret = sscanf(&str[pos], + "%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u %u %lu %lu %lu %lu%n", + &min_sz, &max_sz, &min_nr_a, &max_nr_a, + &min_age, &max_age, &action, "a.ms, + "a.sz, "a.reset_interval, + "a.weight_sz, "a.weight_nr_accesses, + "a.weight_age, &wmarks.metric, + &wmarks.interval, &wmarks.high, &wmarks.mid, + &wmarks.low, &parsed); + if (ret != 18) + break; + if (!damos_action_valid(action)) + goto fail; + + if (min_sz > max_sz || min_nr_a > max_nr_a || min_age > max_age) + goto fail; + + if (wmarks.high < wmarks.mid || wmarks.high < wmarks.low || + wmarks.mid < wmarks.low) + goto fail; + + pos += parsed; + scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a, + min_age, max_age, action, "a, &wmarks); + if (!scheme) + goto fail; + + schemes[*nr_schemes] = scheme; + *nr_schemes += 1; + } + return schemes; +fail: + free_schemes_arr(schemes, *nr_schemes); + return NULL; +} + +static ssize_t dbgfs_schemes_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + struct damos **schemes; + ssize_t nr_schemes = 0, ret; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + schemes = str_to_schemes(kbuf, count, &nr_schemes); + if (!schemes) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + ret = -EBUSY; + goto unlock_out; + } + + ret = damon_set_schemes(ctx, schemes, nr_schemes); + if (!ret) { + ret = count; + nr_schemes = 0; + } + +unlock_out: + mutex_unlock(&ctx->kdamond_lock); + free_schemes_arr(schemes, nr_schemes); +out: + kfree(kbuf); + return ret; +} + +static inline bool targetid_is_pid(const struct damon_ctx *ctx) +{ + return ctx->primitive.target_valid == damon_va_target_valid; +} + +static ssize_t sprint_target_ids(struct damon_ctx *ctx, char *buf, ssize_t len) +{ + struct damon_target *t; + unsigned long id; + int written = 0; + int rc; + + damon_for_each_target(t, ctx) { + id = t->id; + if (targetid_is_pid(ctx)) + /* Show pid numbers to debugfs users */ + id = (unsigned long)pid_vnr((struct pid *)id); + + rc = scnprintf(&buf[written], len - written, "%lu ", id); + if (!rc) + return -ENOMEM; + written += rc; + } + if (written) + written -= 1; + written += scnprintf(&buf[written], len - written, "\n"); + return written; +} + +static ssize_t dbgfs_target_ids_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + ssize_t len; + char ids_buf[320]; + + mutex_lock(&ctx->kdamond_lock); + len = sprint_target_ids(ctx, ids_buf, 320); + mutex_unlock(&ctx->kdamond_lock); + if (len < 0) + return len; + + return simple_read_from_buffer(buf, count, ppos, ids_buf, len); +} + +/* + * Converts a string into an array of unsigned long integers + * + * Returns an array of unsigned long integers if the conversion success, or + * NULL otherwise. + */ +static unsigned long *str_to_target_ids(const char *str, ssize_t len, + ssize_t *nr_ids) +{ + unsigned long *ids; + const int max_nr_ids = 32; + unsigned long id; + int pos = 0, parsed, ret; + + *nr_ids = 0; + ids = kmalloc_array(max_nr_ids, sizeof(id), GFP_KERNEL); + if (!ids) + return NULL; + while (*nr_ids < max_nr_ids && pos < len) { + ret = sscanf(&str[pos], "%lu%n", &id, &parsed); + pos += parsed; + if (ret != 1) + break; + ids[*nr_ids] = id; + *nr_ids += 1; + } + + return ids; +} + +static void dbgfs_put_pids(unsigned long *ids, int nr_ids) +{ + int i; + + for (i = 0; i < nr_ids; i++) + put_pid((struct pid *)ids[i]); +} + +static ssize_t dbgfs_target_ids_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + struct damon_target *t, *next_t; + bool id_is_pid = true; + char *kbuf; + unsigned long *targets; + ssize_t nr_targets; + ssize_t ret; + int i; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + if (!strncmp(kbuf, "paddr\n", count)) { + id_is_pid = false; + /* target id is meaningless here, but we set it just for fun */ + scnprintf(kbuf, count, "42 "); + } + + targets = str_to_target_ids(kbuf, count, &nr_targets); + if (!targets) { + ret = -ENOMEM; + goto out; + } + + if (id_is_pid) { + for (i = 0; i < nr_targets; i++) { + targets[i] = (unsigned long)find_get_pid( + (int)targets[i]); + if (!targets[i]) { + dbgfs_put_pids(targets, i); + ret = -EINVAL; + goto free_targets_out; + } + } + } + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + if (id_is_pid) + dbgfs_put_pids(targets, nr_targets); + ret = -EBUSY; + goto unlock_out; + } + + /* remove previously set targets */ + damon_for_each_target_safe(t, next_t, ctx) { + if (targetid_is_pid(ctx)) + put_pid((struct pid *)t->id); + damon_destroy_target(t); + } + + /* Configure the context for the address space type */ + if (id_is_pid) + damon_va_set_primitives(ctx); + else + damon_pa_set_primitives(ctx); + + ret = damon_set_targets(ctx, targets, nr_targets); + if (ret) { + if (id_is_pid) + dbgfs_put_pids(targets, nr_targets); + } else { + ret = count; + } + +unlock_out: + mutex_unlock(&ctx->kdamond_lock); +free_targets_out: + kfree(targets); +out: + kfree(kbuf); + return ret; +} + +static ssize_t sprint_init_regions(struct damon_ctx *c, char *buf, ssize_t len) +{ + struct damon_target *t; + struct damon_region *r; + int written = 0; + int rc; + + damon_for_each_target(t, c) { + damon_for_each_region(r, t) { + rc = scnprintf(&buf[written], len - written, + "%lu %lu %lu\n", + t->id, r->ar.start, r->ar.end); + if (!rc) + return -ENOMEM; + written += rc; + } + } + return written; +} + +static ssize_t dbgfs_init_regions_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t len; + + kbuf = kmalloc(count, GFP_KERNEL | __GFP_NOWARN); + if (!kbuf) + return -ENOMEM; + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + mutex_unlock(&ctx->kdamond_lock); + len = -EBUSY; + goto out; + } + + len = sprint_init_regions(ctx, kbuf, count); + mutex_unlock(&ctx->kdamond_lock); + if (len < 0) + goto out; + len = simple_read_from_buffer(buf, count, ppos, kbuf, len); + +out: + kfree(kbuf); + return len; +} + +static int add_init_region(struct damon_ctx *c, + unsigned long target_id, struct damon_addr_range *ar) +{ + struct damon_target *t; + struct damon_region *r, *prev; + unsigned long id; + int rc = -EINVAL; + + if (ar->start >= ar->end) + return -EINVAL; + + damon_for_each_target(t, c) { + id = t->id; + if (targetid_is_pid(c)) + id = (unsigned long)pid_vnr((struct pid *)id); + if (id == target_id) { + r = damon_new_region(ar->start, ar->end); + if (!r) + return -ENOMEM; + damon_add_region(r, t); + if (damon_nr_regions(t) > 1) { + prev = damon_prev_region(r); + if (prev->ar.end > r->ar.start) { + damon_destroy_region(r, t); + return -EINVAL; + } + } + rc = 0; + } + } + return rc; +} + +static int set_init_regions(struct damon_ctx *c, const char *str, ssize_t len) +{ + struct damon_target *t; + struct damon_region *r, *next; + int pos = 0, parsed, ret; + unsigned long target_id; + struct damon_addr_range ar; + int err; + + damon_for_each_target(t, c) { + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + } + + while (pos < len) { + ret = sscanf(&str[pos], "%lu %lu %lu%n", + &target_id, &ar.start, &ar.end, &parsed); + if (ret != 3) + break; + err = add_init_region(c, target_id, &ar); + if (err) + goto fail; + pos += parsed; + } + + return 0; + +fail: + damon_for_each_target(t, c) { + damon_for_each_region_safe(r, next, t) + damon_destroy_region(r, t); + } + return err; +} + +static ssize_t dbgfs_init_regions_write(struct file *file, + const char __user *buf, size_t count, + loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t ret = count; + int err; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) { + ret = -EBUSY; + goto unlock_out; + } + + err = set_init_regions(ctx, kbuf, ret); + if (err) + ret = err; + +unlock_out: + mutex_unlock(&ctx->kdamond_lock); + kfree(kbuf); + return ret; +} + +static ssize_t dbgfs_kdamond_pid_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + struct damon_ctx *ctx = file->private_data; + char *kbuf; + ssize_t len; + + kbuf = kmalloc(count, GFP_KERNEL | __GFP_NOWARN); + if (!kbuf) + return -ENOMEM; + + mutex_lock(&ctx->kdamond_lock); + if (ctx->kdamond) + len = scnprintf(kbuf, count, "%d\n", ctx->kdamond->pid); + else + len = scnprintf(kbuf, count, "none\n"); + mutex_unlock(&ctx->kdamond_lock); + if (!len) + goto out; + len = simple_read_from_buffer(buf, count, ppos, kbuf, len); + +out: + kfree(kbuf); + return len; +} + +static int damon_dbgfs_open(struct inode *inode, struct file *file) +{ + file->private_data = inode->i_private; + + return nonseekable_open(inode, file); +} + +static const struct file_operations attrs_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_attrs_read, + .write = dbgfs_attrs_write, +}; + +static const struct file_operations schemes_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_schemes_read, + .write = dbgfs_schemes_write, +}; + +static const struct file_operations target_ids_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_target_ids_read, + .write = dbgfs_target_ids_write, +}; + +static const struct file_operations init_regions_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_init_regions_read, + .write = dbgfs_init_regions_write, +}; + +static const struct file_operations kdamond_pid_fops = { + .open = damon_dbgfs_open, + .read = dbgfs_kdamond_pid_read, +}; + +static void dbgfs_fill_ctx_dir(struct dentry *dir, struct damon_ctx *ctx) +{ + const char * const file_names[] = {"attrs", "schemes", "target_ids", + "init_regions", "kdamond_pid"}; + const struct file_operations *fops[] = {&attrs_fops, &schemes_fops, + &target_ids_fops, &init_regions_fops, &kdamond_pid_fops}; + int i; + + for (i = 0; i < ARRAY_SIZE(file_names); i++) + debugfs_create_file(file_names[i], 0600, dir, ctx, fops[i]); +} + +static void dbgfs_before_terminate(struct damon_ctx *ctx) +{ + struct damon_target *t, *next; + + if (!targetid_is_pid(ctx)) + return; + + mutex_lock(&ctx->kdamond_lock); + damon_for_each_target_safe(t, next, ctx) { + put_pid((struct pid *)t->id); + damon_destroy_target(t); + } + mutex_unlock(&ctx->kdamond_lock); +} + +static struct damon_ctx *dbgfs_new_ctx(void) +{ + struct damon_ctx *ctx; + + ctx = damon_new_ctx(); + if (!ctx) + return NULL; + + damon_va_set_primitives(ctx); + ctx->callback.before_terminate = dbgfs_before_terminate; + return ctx; +} + +static void dbgfs_destroy_ctx(struct damon_ctx *ctx) +{ + damon_destroy_ctx(ctx); +} + +/* + * Make a context of @name and create a debugfs directory for it. + * + * This function should be called while holding damon_dbgfs_lock. + * + * Returns 0 on success, negative error code otherwise. + */ +static int dbgfs_mk_context(char *name) +{ + struct dentry *root, **new_dirs, *new_dir; + struct damon_ctx **new_ctxs, *new_ctx; + + if (damon_nr_running_ctxs()) + return -EBUSY; + + new_ctxs = krealloc(dbgfs_ctxs, sizeof(*dbgfs_ctxs) * + (dbgfs_nr_ctxs + 1), GFP_KERNEL); + if (!new_ctxs) + return -ENOMEM; + dbgfs_ctxs = new_ctxs; + + new_dirs = krealloc(dbgfs_dirs, sizeof(*dbgfs_dirs) * + (dbgfs_nr_ctxs + 1), GFP_KERNEL); + if (!new_dirs) + return -ENOMEM; + dbgfs_dirs = new_dirs; + + root = dbgfs_dirs[0]; + if (!root) + return -ENOENT; + + new_dir = debugfs_create_dir(name, root); + dbgfs_dirs[dbgfs_nr_ctxs] = new_dir; + + new_ctx = dbgfs_new_ctx(); + if (!new_ctx) { + debugfs_remove(new_dir); + dbgfs_dirs[dbgfs_nr_ctxs] = NULL; + return -ENOMEM; + } + + dbgfs_ctxs[dbgfs_nr_ctxs] = new_ctx; + dbgfs_fill_ctx_dir(dbgfs_dirs[dbgfs_nr_ctxs], + dbgfs_ctxs[dbgfs_nr_ctxs]); + dbgfs_nr_ctxs++; + + return 0; +} + +static ssize_t dbgfs_mk_context_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + char *kbuf; + char *ctx_name; + ssize_t ret; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + ctx_name = kmalloc(count + 1, GFP_KERNEL); + if (!ctx_name) { + kfree(kbuf); + return -ENOMEM; + } + + /* Trim white space */ + if (sscanf(kbuf, "%s", ctx_name) != 1) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&damon_dbgfs_lock); + ret = dbgfs_mk_context(ctx_name); + if (!ret) + ret = count; + mutex_unlock(&damon_dbgfs_lock); + +out: + kfree(kbuf); + kfree(ctx_name); + return ret; +} + +/* + * Remove a context of @name and its debugfs directory. + * + * This function should be called while holding damon_dbgfs_lock. + * + * Return 0 on success, negative error code otherwise. + */ +static int dbgfs_rm_context(char *name) +{ + struct dentry *root, *dir, **new_dirs; + struct damon_ctx **new_ctxs; + int i, j; + + if (damon_nr_running_ctxs()) + return -EBUSY; + + root = dbgfs_dirs[0]; + if (!root) + return -ENOENT; + + dir = debugfs_lookup(name, root); + if (!dir) + return -ENOENT; + + new_dirs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_dirs), + GFP_KERNEL); + if (!new_dirs) + return -ENOMEM; + + new_ctxs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_ctxs), + GFP_KERNEL); + if (!new_ctxs) { + kfree(new_dirs); + return -ENOMEM; + } + + for (i = 0, j = 0; i < dbgfs_nr_ctxs; i++) { + if (dbgfs_dirs[i] == dir) { + debugfs_remove(dbgfs_dirs[i]); + dbgfs_destroy_ctx(dbgfs_ctxs[i]); + continue; + } + new_dirs[j] = dbgfs_dirs[i]; + new_ctxs[j++] = dbgfs_ctxs[i]; + } + + kfree(dbgfs_dirs); + kfree(dbgfs_ctxs); + + dbgfs_dirs = new_dirs; + dbgfs_ctxs = new_ctxs; + dbgfs_nr_ctxs--; + + return 0; +} + +static ssize_t dbgfs_rm_context_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + char *kbuf; + ssize_t ret; + char *ctx_name; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + ctx_name = kmalloc(count + 1, GFP_KERNEL); + if (!ctx_name) { + kfree(kbuf); + return -ENOMEM; + } + + /* Trim white space */ + if (sscanf(kbuf, "%s", ctx_name) != 1) { + ret = -EINVAL; + goto out; + } + + mutex_lock(&damon_dbgfs_lock); + ret = dbgfs_rm_context(ctx_name); + if (!ret) + ret = count; + mutex_unlock(&damon_dbgfs_lock); + +out: + kfree(kbuf); + kfree(ctx_name); + return ret; +} + +static ssize_t dbgfs_monitor_on_read(struct file *file, + char __user *buf, size_t count, loff_t *ppos) +{ + char monitor_on_buf[5]; + bool monitor_on = damon_nr_running_ctxs() != 0; + int len; + + len = scnprintf(monitor_on_buf, 5, monitor_on ? "on\n" : "off\n"); + + return simple_read_from_buffer(buf, count, ppos, monitor_on_buf, len); +} + +static ssize_t dbgfs_monitor_on_write(struct file *file, + const char __user *buf, size_t count, loff_t *ppos) +{ + ssize_t ret; + char *kbuf; + + kbuf = user_input_str(buf, count, ppos); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + /* Remove white space */ + if (sscanf(kbuf, "%s", kbuf) != 1) { + kfree(kbuf); + return -EINVAL; + } + + mutex_lock(&damon_dbgfs_lock); + if (!strncmp(kbuf, "on", count)) { + int i; + + for (i = 0; i < dbgfs_nr_ctxs; i++) { + if (damon_targets_empty(dbgfs_ctxs[i])) { + kfree(kbuf); + mutex_unlock(&damon_dbgfs_lock); + return -EINVAL; + } + } + ret = damon_start(dbgfs_ctxs, dbgfs_nr_ctxs); + } else if (!strncmp(kbuf, "off", count)) { + ret = damon_stop(dbgfs_ctxs, dbgfs_nr_ctxs); + } else { + ret = -EINVAL; + } + mutex_unlock(&damon_dbgfs_lock); + + if (!ret) + ret = count; + kfree(kbuf); + return ret; +} + +static const struct file_operations mk_contexts_fops = { + .write = dbgfs_mk_context_write, +}; + +static const struct file_operations rm_contexts_fops = { + .write = dbgfs_rm_context_write, +}; + +static const struct file_operations monitor_on_fops = { + .read = dbgfs_monitor_on_read, + .write = dbgfs_monitor_on_write, +}; + +static int __init __damon_dbgfs_init(void) +{ + struct dentry *dbgfs_root; + const char * const file_names[] = {"mk_contexts", "rm_contexts", + "monitor_on"}; + const struct file_operations *fops[] = {&mk_contexts_fops, + &rm_contexts_fops, &monitor_on_fops}; + int i; + + dbgfs_root = debugfs_create_dir("damon", NULL); + + for (i = 0; i < ARRAY_SIZE(file_names); i++) + debugfs_create_file(file_names[i], 0600, dbgfs_root, NULL, + fops[i]); + dbgfs_fill_ctx_dir(dbgfs_root, dbgfs_ctxs[0]); + + dbgfs_dirs = kmalloc_array(1, sizeof(dbgfs_root), GFP_KERNEL); + if (!dbgfs_dirs) { + debugfs_remove(dbgfs_root); + return -ENOMEM; + } + dbgfs_dirs[0] = dbgfs_root; + + return 0; +} + +/* + * Functions for the initialization + */ + +static int __init damon_dbgfs_init(void) +{ + int rc = -ENOMEM; + + mutex_lock(&damon_dbgfs_lock); + dbgfs_ctxs = kmalloc(sizeof(*dbgfs_ctxs), GFP_KERNEL); + if (!dbgfs_ctxs) + goto out; + dbgfs_ctxs[0] = dbgfs_new_ctx(); + if (!dbgfs_ctxs[0]) { + kfree(dbgfs_ctxs); + goto out; + } + dbgfs_nr_ctxs = 1; + + rc = __damon_dbgfs_init(); + if (rc) { + kfree(dbgfs_ctxs[0]); + kfree(dbgfs_ctxs); + pr_err("%s: dbgfs init failed\n", __func__); + } + +out: + mutex_unlock(&damon_dbgfs_lock); + return rc; +} + +module_init(damon_dbgfs_init); + +#include "dbgfs-test.h" diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c new file mode 100644 index 000000000000..5e8244f65a1a --- /dev/null +++ b/mm/damon/paddr.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON Primitives for The Physical Address Space + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-pa: " fmt + +#include +#include +#include +#include +#include + +#include "../internal.h" +#include "prmtv-common.h" + +static bool __damon_pa_mkold(struct page *page, struct vm_area_struct *vma, + unsigned long addr, void *arg) +{ + struct page_vma_mapped_walk pvmw = { + .page = page, + .vma = vma, + .address = addr, + }; + + while (page_vma_mapped_walk(&pvmw)) { + addr = pvmw.address; + if (pvmw.pte) + damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr); + else + damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr); + } + return true; +} + +static void damon_pa_mkold(unsigned long paddr) +{ + struct page *page = damon_get_page(PHYS_PFN(paddr)); + struct rmap_walk_control rwc = { + .rmap_one = __damon_pa_mkold, + .anon_lock = page_lock_anon_vma_read, + }; + bool need_lock; + + if (!page) + return; + + if (!page_mapped(page) || !page_rmapping(page)) { + set_page_idle(page); + goto out; + } + + need_lock = !PageAnon(page) || PageKsm(page); + if (need_lock && !trylock_page(page)) + goto out; + + rmap_walk(page, &rwc); + + if (need_lock) + unlock_page(page); + +out: + put_page(page); +} + +static void __damon_pa_prepare_access_check(struct damon_ctx *ctx, + struct damon_region *r) +{ + r->sampling_addr = damon_rand(r->ar.start, r->ar.end); + + damon_pa_mkold(r->sampling_addr); +} + +static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct damon_region *r; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) + __damon_pa_prepare_access_check(ctx, r); + } +} + +struct damon_pa_access_chk_result { + unsigned long page_sz; + bool accessed; +}; + +static bool __damon_pa_young(struct page *page, struct vm_area_struct *vma, + unsigned long addr, void *arg) +{ + struct damon_pa_access_chk_result *result = arg; + struct page_vma_mapped_walk pvmw = { + .page = page, + .vma = vma, + .address = addr, + }; + + result->accessed = false; + result->page_sz = PAGE_SIZE; + while (page_vma_mapped_walk(&pvmw)) { + addr = pvmw.address; + if (pvmw.pte) { + result->accessed = pte_young(*pvmw.pte) || + !page_is_idle(page) || + mmu_notifier_test_young(vma->vm_mm, addr); + } else { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + result->accessed = pmd_young(*pvmw.pmd) || + !page_is_idle(page) || + mmu_notifier_test_young(vma->vm_mm, addr); + result->page_sz = ((1UL) << HPAGE_PMD_SHIFT); +#else + WARN_ON_ONCE(1); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + } + if (result->accessed) { + page_vma_mapped_walk_done(&pvmw); + break; + } + } + + /* If accessed, stop walking */ + return !result->accessed; +} + +static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz) +{ + struct page *page = damon_get_page(PHYS_PFN(paddr)); + struct damon_pa_access_chk_result result = { + .page_sz = PAGE_SIZE, + .accessed = false, + }; + struct rmap_walk_control rwc = { + .arg = &result, + .rmap_one = __damon_pa_young, + .anon_lock = page_lock_anon_vma_read, + }; + bool need_lock; + + if (!page) + return false; + + if (!page_mapped(page) || !page_rmapping(page)) { + if (page_is_idle(page)) + result.accessed = false; + else + result.accessed = true; + put_page(page); + goto out; + } + + need_lock = !PageAnon(page) || PageKsm(page); + if (need_lock && !trylock_page(page)) { + put_page(page); + return NULL; + } + + rmap_walk(page, &rwc); + + if (need_lock) + unlock_page(page); + put_page(page); + +out: + *page_sz = result.page_sz; + return result.accessed; +} + +static void __damon_pa_check_access(struct damon_ctx *ctx, + struct damon_region *r) +{ + static unsigned long last_addr; + static unsigned long last_page_sz = PAGE_SIZE; + static bool last_accessed; + + /* If the region is in the last checked page, reuse the result */ + if (ALIGN_DOWN(last_addr, last_page_sz) == + ALIGN_DOWN(r->sampling_addr, last_page_sz)) { + if (last_accessed) + r->nr_accesses++; + return; + } + + last_accessed = damon_pa_young(r->sampling_addr, &last_page_sz); + if (last_accessed) + r->nr_accesses++; + + last_addr = r->sampling_addr; +} + +static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct damon_region *r; + unsigned int max_nr_accesses = 0; + + damon_for_each_target(t, ctx) { + damon_for_each_region(r, t) { + __damon_pa_check_access(ctx, r); + max_nr_accesses = max(r->nr_accesses, max_nr_accesses); + } + } + + return max_nr_accesses; +} + +bool damon_pa_target_valid(void *t) +{ + return true; +} + +static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) +{ + unsigned long addr, applied; + LIST_HEAD(page_list); + + if (scheme->action != DAMOS_PAGEOUT) + return 0; + + for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { + struct page *page = damon_get_page(PHYS_PFN(addr)); + + if (!page) + continue; + + ClearPageReferenced(page); + test_and_clear_page_young(page); + if (isolate_lru_page(page)) { + put_page(page); + continue; + } + if (PageUnevictable(page)) { + putback_lru_page(page); + } else { + list_add(&page->lru, &page_list); + put_page(page); + } + } + applied = reclaim_pages(&page_list); + cond_resched(); + return applied * PAGE_SIZE; +} + +static int damon_pa_scheme_score(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) +{ + switch (scheme->action) { + case DAMOS_PAGEOUT: + return damon_pageout_score(context, r, scheme); + default: + break; + } + + return DAMOS_MAX_SCORE; +} + +void damon_pa_set_primitives(struct damon_ctx *ctx) +{ + ctx->primitive.init = NULL; + ctx->primitive.update = NULL; + ctx->primitive.prepare_access_checks = damon_pa_prepare_access_checks; + ctx->primitive.check_accesses = damon_pa_check_accesses; + ctx->primitive.reset_aggregated = NULL; + ctx->primitive.target_valid = damon_pa_target_valid; + ctx->primitive.cleanup = NULL; + ctx->primitive.apply_scheme = damon_pa_apply_scheme; + ctx->primitive.get_scheme_score = damon_pa_scheme_score; +} diff --git a/mm/damon/prmtv-common.c b/mm/damon/prmtv-common.c new file mode 100644 index 000000000000..92a04f5831d6 --- /dev/null +++ b/mm/damon/prmtv-common.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common Primitives for Data Access Monitoring + * + * Author: SeongJae Park + */ + +#include +#include +#include +#include + +#include "prmtv-common.h" + +/* + * Get an online page for a pfn if it's in the LRU list. Otherwise, returns + * NULL. + * + * The body of this function is stolen from the 'page_idle_get_page()'. We + * steal rather than reuse it because the code is quite simple. + */ +struct page *damon_get_page(unsigned long pfn) +{ + struct page *page = pfn_to_online_page(pfn); + + if (!page || !PageLRU(page) || !get_page_unless_zero(page)) + return NULL; + + if (unlikely(!PageLRU(page))) { + put_page(page); + page = NULL; + } + return page; +} + +void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr) +{ + bool referenced = false; + struct page *page = damon_get_page(pte_pfn(*pte)); + + if (!page) + return; + + if (pte_young(*pte)) { + referenced = true; + *pte = pte_mkold(*pte); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, addr + PAGE_SIZE)) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +} + +void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + bool referenced = false; + struct page *page = damon_get_page(pmd_pfn(*pmd)); + + if (!page) + return; + + if (pmd_young(*pmd)) { + referenced = true; + *pmd = pmd_mkold(*pmd); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, + addr + ((1UL) << HPAGE_PMD_SHIFT))) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +} + +#define DAMON_MAX_SUBSCORE (100) +#define DAMON_MAX_AGE_IN_LOG (32) + +int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s) +{ + unsigned int max_nr_accesses; + int freq_subscore; + unsigned int age_in_sec; + int age_in_log, age_subscore; + unsigned int freq_weight = s->quota.weight_nr_accesses; + unsigned int age_weight = s->quota.weight_age; + int hotness; + + max_nr_accesses = c->aggr_interval / c->sample_interval; + freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses; + + age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000; + for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; + age_in_log++, age_in_sec >>= 1) + ; + + /* If frequency is 0, higher age means it's colder */ + if (freq_subscore == 0) + age_in_log *= -1; + + /* + * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. + * Scale it to be in [0, 100] and set it as age subscore. + */ + age_in_log += DAMON_MAX_AGE_IN_LOG; + age_subscore = age_in_log * DAMON_MAX_SUBSCORE / + DAMON_MAX_AGE_IN_LOG / 2; + + hotness = (freq_weight * freq_subscore + age_weight * age_subscore); + if (freq_weight + age_weight) + hotness /= freq_weight + age_weight; + /* + * Transform it to fit in [0, DAMOS_MAX_SCORE] + */ + hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; + + /* Return coldness of the region */ + return DAMOS_MAX_SCORE - hotness; +} diff --git a/mm/damon/prmtv-common.h b/mm/damon/prmtv-common.h new file mode 100644 index 000000000000..e790cb5f8fe0 --- /dev/null +++ b/mm/damon/prmtv-common.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common Primitives for Data Access Monitoring + * + * Author: SeongJae Park + */ + +#include + +struct page *damon_get_page(unsigned long pfn); + +void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr); +void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr); + +int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, + struct damos *s); diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c new file mode 100644 index 000000000000..183d4f3b4fde --- /dev/null +++ b/mm/damon/reclaim.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON-based page reclamation + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-reclaim: " fmt + +#include +#include +#include +#include +#include + +#ifdef MODULE_PARAM_PREFIX +#undef MODULE_PARAM_PREFIX +#endif +#define MODULE_PARAM_PREFIX "damon_reclaim." + +/* + * Enable or disable DAMON_RECLAIM. + * + * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. + * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could + * do no real monitoring and reclamation due to the watermarks-based activation + * condition. Refer to below descriptions for the watermarks parameter for + * this. + */ +static bool enabled __read_mostly; + +/* + * Time threshold for cold memory regions identification in microseconds. + * + * If a memory region is not accessed for this or longer time, DAMON_RECLAIM + * identifies the region as cold, and reclaims. 120 seconds by default. + */ +static unsigned long min_age __read_mostly = 120000000; +module_param(min_age, ulong, 0600); + +/* + * Limit of time for trying the reclamation in milliseconds. + * + * DAMON_RECLAIM tries to use only up to this time within a time window + * (quota_reset_interval_ms) for trying reclamation of cold pages. This can be + * used for limiting CPU consumption of DAMON_RECLAIM. If the value is zero, + * the limit is disabled. + * + * 10 ms by default. + */ +static unsigned long quota_ms __read_mostly = 10; +module_param(quota_ms, ulong, 0600); + +/* + * Limit of size of memory for the reclamation in bytes. + * + * DAMON_RECLAIM charges amount of memory which it tried to reclaim within a + * time window (quota_reset_interval_ms) and makes no more than this limit is + * tried. This can be used for limiting consumption of CPU and IO. If this + * value is zero, the limit is disabled. + * + * 128 MiB by default. + */ +static unsigned long quota_sz __read_mostly = 128 * 1024 * 1024; +module_param(quota_sz, ulong, 0600); + +/* + * The time/size quota charge reset interval in milliseconds. + * + * The charge reset interval for the quota of time (quota_ms) and size + * (quota_sz). That is, DAMON_RECLAIM does not try reclamation for more than + * quota_ms milliseconds or quota_sz bytes within quota_reset_interval_ms + * milliseconds. + * + * 1 second by default. + */ +static unsigned long quota_reset_interval_ms __read_mostly = 1000; +module_param(quota_reset_interval_ms, ulong, 0600); + +/* + * The watermarks check time interval in microseconds. + * + * Minimal time to wait before checking the watermarks, when DAMON_RECLAIM is + * enabled but inactive due to its watermarks rule. 5 seconds by default. + */ +static unsigned long wmarks_interval __read_mostly = 5000000; +module_param(wmarks_interval, ulong, 0600); + +/* + * Free memory rate (per thousand) for the high watermark. + * + * If free memory of the system in bytes per thousand bytes is higher than + * this, DAMON_RECLAIM becomes inactive, so it does nothing but periodically + * checks the watermarks. 500 (50%) by default. + */ +static unsigned long wmarks_high __read_mostly = 500; +module_param(wmarks_high, ulong, 0600); + +/* + * Free memory rate (per thousand) for the middle watermark. + * + * If free memory of the system in bytes per thousand bytes is between this and + * the low watermark, DAMON_RECLAIM becomes active, so starts the monitoring + * and the reclaiming. 400 (40%) by default. + */ +static unsigned long wmarks_mid __read_mostly = 400; +module_param(wmarks_mid, ulong, 0600); + +/* + * Free memory rate (per thousand) for the low watermark. + * + * If free memory of the system in bytes per thousand bytes is lower than this, + * DAMON_RECLAIM becomes inactive, so it does nothing but periodically checks + * the watermarks. In the case, the system falls back to the LRU-based page + * granularity reclamation logic. 200 (20%) by default. + */ +static unsigned long wmarks_low __read_mostly = 200; +module_param(wmarks_low, ulong, 0600); + +/* + * Sampling interval for the monitoring in microseconds. + * + * The sampling interval of DAMON for the cold memory monitoring. Please refer + * to the DAMON documentation for more detail. 5 ms by default. + */ +static unsigned long sample_interval __read_mostly = 5000; +module_param(sample_interval, ulong, 0600); + +/* + * Aggregation interval for the monitoring in microseconds. + * + * The aggregation interval of DAMON for the cold memory monitoring. Please + * refer to the DAMON documentation for more detail. 100 ms by default. + */ +static unsigned long aggr_interval __read_mostly = 100000; +module_param(aggr_interval, ulong, 0600); + +/* + * Minimum number of monitoring regions. + * + * The minimal number of monitoring regions of DAMON for the cold memory + * monitoring. This can be used to set lower-bound of the monitoring quality. + * But, setting this too high could result in increased monitoring overhead. + * Please refer to the DAMON documentation for more detail. 10 by default. + */ +static unsigned long min_nr_regions __read_mostly = 10; +module_param(min_nr_regions, ulong, 0600); + +/* + * Maximum number of monitoring regions. + * + * The maximum number of monitoring regions of DAMON for the cold memory + * monitoring. This can be used to set upper-bound of the monitoring overhead. + * However, setting this too low could result in bad monitoring quality. + * Please refer to the DAMON documentation for more detail. 1000 by default. + */ +static unsigned long max_nr_regions __read_mostly = 1000; +module_param(max_nr_regions, ulong, 0600); + +/* + * Start of the target memory region in physical address. + * + * The start physical address of memory region that DAMON_RECLAIM will do work + * against. By default, biggest System RAM is used as the region. + */ +static unsigned long monitor_region_start __read_mostly; +module_param(monitor_region_start, ulong, 0600); + +/* + * End of the target memory region in physical address. + * + * The end physical address of memory region that DAMON_RECLAIM will do work + * against. By default, biggest System RAM is used as the region. + */ +static unsigned long monitor_region_end __read_mostly; +module_param(monitor_region_end, ulong, 0600); + +/* + * PID of the DAMON thread + * + * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. + * Else, -1. + */ +static int kdamond_pid __read_mostly = -1; +module_param(kdamond_pid, int, 0400); + +/* + * Number of memory regions that tried to be reclaimed. + */ +static unsigned long nr_reclaim_tried_regions __read_mostly; +module_param(nr_reclaim_tried_regions, ulong, 0400); + +/* + * Total bytes of memory regions that tried to be reclaimed. + */ +static unsigned long bytes_reclaim_tried_regions __read_mostly; +module_param(bytes_reclaim_tried_regions, ulong, 0400); + +/* + * Number of memory regions that successfully be reclaimed. + */ +static unsigned long nr_reclaimed_regions __read_mostly; +module_param(nr_reclaimed_regions, ulong, 0400); + +/* + * Total bytes of memory regions that successfully be reclaimed. + */ +static unsigned long bytes_reclaimed_regions __read_mostly; +module_param(bytes_reclaimed_regions, ulong, 0400); + +/* + * Number of times that the time/space quota limits have exceeded + */ +static unsigned long nr_quota_exceeds __read_mostly; +module_param(nr_quota_exceeds, ulong, 0400); + +static struct damon_ctx *ctx; +static struct damon_target *target; + +struct damon_reclaim_ram_walk_arg { + unsigned long start; + unsigned long end; +}; + +static int walk_system_ram(struct resource *res, void *arg) +{ + struct damon_reclaim_ram_walk_arg *a = arg; + + if (a->end - a->start < res->end - res->start) { + a->start = res->start; + a->end = res->end; + } + return 0; +} + +/* + * Find biggest 'System RAM' resource and store its start and end address in + * @start and @end, respectively. If no System RAM is found, returns false. + */ +static bool get_monitoring_region(unsigned long *start, unsigned long *end) +{ + struct damon_reclaim_ram_walk_arg arg = {}; + + walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); + if (arg.end <= arg.start) + return false; + + *start = arg.start; + *end = arg.end; + return true; +} + +static struct damos *damon_reclaim_new_scheme(void) +{ + struct damos_watermarks wmarks = { + .metric = DAMOS_WMARK_FREE_MEM_RATE, + .interval = wmarks_interval, + .high = wmarks_high, + .mid = wmarks_mid, + .low = wmarks_low, + }; + struct damos_quota quota = { + /* + * Do not try reclamation for more than quota_ms milliseconds + * or quota_sz bytes within quota_reset_interval_ms. + */ + .ms = quota_ms, + .sz = quota_sz, + .reset_interval = quota_reset_interval_ms, + /* Within the quota, page out older regions first. */ + .weight_sz = 0, + .weight_nr_accesses = 0, + .weight_age = 1 + }; + struct damos *scheme = damon_new_scheme( + /* Find regions having PAGE_SIZE or larger size */ + PAGE_SIZE, ULONG_MAX, + /* and not accessed at all */ + 0, 0, + /* for min_age or more micro-seconds, and */ + min_age / aggr_interval, UINT_MAX, + /* page out those, as soon as found */ + DAMOS_PAGEOUT, + /* under the quota. */ + "a, + /* (De)activate this according to the watermarks. */ + &wmarks); + + return scheme; +} + +static int damon_reclaim_turn(bool on) +{ + struct damon_region *region; + struct damos *scheme; + int err; + + if (!on) { + err = damon_stop(&ctx, 1); + if (!err) + kdamond_pid = -1; + return err; + } + + err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0, + min_nr_regions, max_nr_regions); + if (err) + return err; + + if (monitor_region_start > monitor_region_end) + return -EINVAL; + if (!monitor_region_start && !monitor_region_end && + !get_monitoring_region(&monitor_region_start, + &monitor_region_end)) + return -EINVAL; + /* DAMON will free this on its own when finish monitoring */ + region = damon_new_region(monitor_region_start, monitor_region_end); + if (!region) + return -ENOMEM; + damon_add_region(region, target); + + /* Will be freed by 'damon_set_schemes()' below */ + scheme = damon_reclaim_new_scheme(); + if (!scheme) { + err = -ENOMEM; + goto free_region_out; + } + err = damon_set_schemes(ctx, &scheme, 1); + if (err) + goto free_scheme_out; + + err = damon_start(&ctx, 1); + if (!err) { + kdamond_pid = ctx->kdamond->pid; + return 0; + } + +free_scheme_out: + damon_destroy_scheme(scheme); +free_region_out: + damon_destroy_region(region, target); + return err; +} + +#define ENABLE_CHECK_INTERVAL_MS 1000 +static struct delayed_work damon_reclaim_timer; +static void damon_reclaim_timer_fn(struct work_struct *work) +{ + static bool last_enabled; + bool now_enabled; + + now_enabled = enabled; + if (last_enabled != now_enabled) { + if (!damon_reclaim_turn(now_enabled)) + last_enabled = now_enabled; + else + enabled = last_enabled; + } + + if (enabled) + schedule_delayed_work(&damon_reclaim_timer, + msecs_to_jiffies(ENABLE_CHECK_INTERVAL_MS)); +} +static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn); + +static int enabled_store(const char *val, + const struct kernel_param *kp) +{ + int rc = param_set_bool(val, kp); + + if (rc < 0) + return rc; + + if (enabled) + schedule_delayed_work(&damon_reclaim_timer, 0); + + return 0; +} + +static const struct kernel_param_ops enabled_param_ops = { + .set = enabled_store, + .get = param_get_bool, +}; + +module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); +MODULE_PARM_DESC(enabled, + "Enable or disable DAMON_RECLAIM (default: disabled)"); + +static int damon_reclaim_after_aggregation(struct damon_ctx *c) +{ + struct damos *s; + + /* update the stats parameter */ + damon_for_each_scheme(s, c) { + nr_reclaim_tried_regions = s->stat.nr_tried; + bytes_reclaim_tried_regions = s->stat.sz_tried; + nr_reclaimed_regions = s->stat.nr_applied; + bytes_reclaimed_regions = s->stat.sz_applied; + nr_quota_exceeds = s->stat.qt_exceeds; + } + return 0; +} + +static int __init damon_reclaim_init(void) +{ + ctx = damon_new_ctx(); + if (!ctx) + return -ENOMEM; + + damon_pa_set_primitives(ctx); + ctx->callback.after_aggregation = damon_reclaim_after_aggregation; + + /* 4242 means nothing but fun */ + target = damon_new_target(4242); + if (!target) { + damon_destroy_ctx(ctx); + return -ENOMEM; + } + damon_add_target(ctx, target); + + schedule_delayed_work(&damon_reclaim_timer, 0); + return 0; +} + +module_init(damon_reclaim_init); diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h new file mode 100644 index 000000000000..6a1b9272ea12 --- /dev/null +++ b/mm/damon/vaddr-test.h @@ -0,0 +1,324 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Data Access Monitor Unit Tests + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Author: SeongJae Park + */ + +#ifdef CONFIG_DAMON_VADDR_KUNIT_TEST + +#ifndef _DAMON_VADDR_TEST_H +#define _DAMON_VADDR_TEST_H + +#include + +static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) +{ + int i, j; + unsigned long largest_gap, gap; + + if (!nr_vmas) + return; + + for (i = 0; i < nr_vmas - 1; i++) { + vmas[i].vm_next = &vmas[i + 1]; + + vmas[i].vm_rb.rb_left = NULL; + vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb; + + largest_gap = 0; + for (j = i; j < nr_vmas; j++) { + if (j == 0) + continue; + gap = vmas[j].vm_start - vmas[j - 1].vm_end; + if (gap > largest_gap) + largest_gap = gap; + } + vmas[i].rb_subtree_gap = largest_gap; + } + vmas[i].vm_next = NULL; + vmas[i].vm_rb.rb_right = NULL; + vmas[i].rb_subtree_gap = 0; +} + +/* + * Test __damon_va_three_regions() function + * + * In case of virtual memory address spaces monitoring, DAMON converts the + * complex and dynamic memory mappings of each target task to three + * discontiguous regions which cover every mapped areas. However, the three + * regions should not include the two biggest unmapped areas in the original + * mapping, because the two biggest areas are normally the areas between 1) + * heap and the mmap()-ed regions, and 2) the mmap()-ed regions and stack. + * Because these two unmapped areas are very huge but obviously never accessed, + * covering the region is just a waste. + * + * '__damon_va_three_regions() receives an address space of a process. It + * first identifies the start of mappings, end of mappings, and the two biggest + * unmapped areas. After that, based on the information, it constructs the + * three regions and returns. For more detail, refer to the comment of + * 'damon_init_regions_of()' function definition in 'mm/damon.c' file. + * + * For example, suppose virtual address ranges of 10-20, 20-25, 200-210, + * 210-220, 300-305, and 307-330 (Other comments represent this mappings in + * more short form: 10-20-25, 200-210-220, 300-305, 307-330) of a process are + * mapped. To cover every mappings, the three regions should start with 10, + * and end with 305. The process also has three unmapped areas, 25-200, + * 220-300, and 305-307. Among those, 25-200 and 220-300 are the biggest two + * unmapped areas, and thus it should be converted to three regions of 10-25, + * 200-220, and 300-330. + */ +static void damon_test_three_regions_in_vmas(struct kunit *test) +{ + struct damon_addr_range regions[3] = {0,}; + /* 10-20-25, 200-210-220, 300-305, 307-330 */ + struct vm_area_struct vmas[] = { + (struct vm_area_struct) {.vm_start = 10, .vm_end = 20}, + (struct vm_area_struct) {.vm_start = 20, .vm_end = 25}, + (struct vm_area_struct) {.vm_start = 200, .vm_end = 210}, + (struct vm_area_struct) {.vm_start = 210, .vm_end = 220}, + (struct vm_area_struct) {.vm_start = 300, .vm_end = 305}, + (struct vm_area_struct) {.vm_start = 307, .vm_end = 330}, + }; + + __link_vmas(vmas, 6); + + __damon_va_three_regions(&vmas[0], regions); + + KUNIT_EXPECT_EQ(test, 10ul, regions[0].start); + KUNIT_EXPECT_EQ(test, 25ul, regions[0].end); + KUNIT_EXPECT_EQ(test, 200ul, regions[1].start); + KUNIT_EXPECT_EQ(test, 220ul, regions[1].end); + KUNIT_EXPECT_EQ(test, 300ul, regions[2].start); + KUNIT_EXPECT_EQ(test, 330ul, regions[2].end); +} + +static struct damon_region *__nth_region_of(struct damon_target *t, int idx) +{ + struct damon_region *r; + unsigned int i = 0; + + damon_for_each_region(r, t) { + if (i++ == idx) + return r; + } + + return NULL; +} + +/* + * Test 'damon_va_apply_three_regions()' + * + * test kunit object + * regions an array containing start/end addresses of current + * monitoring target regions + * nr_regions the number of the addresses in 'regions' + * three_regions The three regions that need to be applied now + * expected start/end addresses of monitoring target regions that + * 'three_regions' are applied + * nr_expected the number of addresses in 'expected' + * + * The memory mapping of the target processes changes dynamically. To follow + * the change, DAMON periodically reads the mappings, simplifies it to the + * three regions, and updates the monitoring target regions to fit in the three + * regions. The update of current target regions is the role of + * 'damon_va_apply_three_regions()'. + * + * This test passes the given target regions and the new three regions that + * need to be applied to the function and check whether it updates the regions + * as expected. + */ +static void damon_do_test_apply_three_regions(struct kunit *test, + unsigned long *regions, int nr_regions, + struct damon_addr_range *three_regions, + unsigned long *expected, int nr_expected) +{ + struct damon_target *t; + struct damon_region *r; + int i; + + t = damon_new_target(42); + for (i = 0; i < nr_regions / 2; i++) { + r = damon_new_region(regions[i * 2], regions[i * 2 + 1]); + damon_add_region(r, t); + } + + damon_va_apply_three_regions(t, three_regions); + + for (i = 0; i < nr_expected / 2; i++) { + r = __nth_region_of(t, i); + KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); + KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); + } +} + +/* + * This function test most common case where the three big regions are only + * slightly changed. Target regions should adjust their boundary (10-20-30, + * 50-55, 70-80, 90-100) to fit with the new big regions or remove target + * regions (57-79) that now out of the three regions. + */ +static void damon_test_apply_three_regions1(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-27, 45-55, 73-104 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 27}, + (struct damon_addr_range){.start = 45, .end = 55}, + (struct damon_addr_range){.start = 73, .end = 104} }; + /* 5-20-27, 45-55, 73-80-90-104 */ + unsigned long expected[] = {5, 20, 20, 27, 45, 55, + 73, 80, 80, 90, 90, 104}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +/* + * Test slightly bigger change. Similar to above, but the second big region + * now require two target regions (50-55, 57-59) to be removed. + */ +static void damon_test_apply_three_regions2(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-27, 56-57, 65-104 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 27}, + (struct damon_addr_range){.start = 56, .end = 57}, + (struct damon_addr_range){.start = 65, .end = 104} }; + /* 5-20-27, 56-57, 65-80-90-104 */ + unsigned long expected[] = {5, 20, 20, 27, 56, 57, + 65, 80, 80, 90, 90, 104}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +/* + * Test a big change. The second big region has totally freed and mapped to + * different area (50-59 -> 61-63). The target regions which were in the old + * second big region (50-55-57-59) should be removed and new target region + * covering the second big region (61-63) should be created. + */ +static void damon_test_apply_three_regions3(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-27, 61-63, 65-104 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 27}, + (struct damon_addr_range){.start = 61, .end = 63}, + (struct damon_addr_range){.start = 65, .end = 104} }; + /* 5-20-27, 61-63, 65-80-90-104 */ + unsigned long expected[] = {5, 20, 20, 27, 61, 63, + 65, 80, 80, 90, 90, 104}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +/* + * Test another big change. Both of the second and third big regions (50-59 + * and 70-100) has totally freed and mapped to different area (30-32 and + * 65-68). The target regions which were in the old second and third big + * regions should now be removed and new target regions covering the new second + * and third big regions should be created. + */ +static void damon_test_apply_three_regions4(struct kunit *test) +{ + /* 10-20-30, 50-55-57-59, 70-80-90-100 */ + unsigned long regions[] = {10, 20, 20, 30, 50, 55, 55, 57, 57, 59, + 70, 80, 80, 90, 90, 100}; + /* 5-7, 30-32, 65-68 */ + struct damon_addr_range new_three_regions[3] = { + (struct damon_addr_range){.start = 5, .end = 7}, + (struct damon_addr_range){.start = 30, .end = 32}, + (struct damon_addr_range){.start = 65, .end = 68} }; + /* expect 5-7, 30-32, 65-68 */ + unsigned long expected[] = {5, 7, 30, 32, 65, 68}; + + damon_do_test_apply_three_regions(test, regions, ARRAY_SIZE(regions), + new_three_regions, expected, ARRAY_SIZE(expected)); +} + +static void damon_test_split_evenly_fail(struct kunit *test, + unsigned long start, unsigned long end, unsigned int nr_pieces) +{ + struct damon_target *t = damon_new_target(42); + struct damon_region *r = damon_new_region(start, end); + + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, + damon_va_evenly_split_region(t, r, nr_pieces), -EINVAL); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 1u); + + damon_for_each_region(r, t) { + KUNIT_EXPECT_EQ(test, r->ar.start, start); + KUNIT_EXPECT_EQ(test, r->ar.end, end); + } + + damon_free_target(t); +} + +static void damon_test_split_evenly_succ(struct kunit *test, + unsigned long start, unsigned long end, unsigned int nr_pieces) +{ + struct damon_target *t = damon_new_target(42); + struct damon_region *r = damon_new_region(start, end); + unsigned long expected_width = (end - start) / nr_pieces; + unsigned long i = 0; + + damon_add_region(r, t); + KUNIT_EXPECT_EQ(test, + damon_va_evenly_split_region(t, r, nr_pieces), 0); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_pieces); + + damon_for_each_region(r, t) { + if (i == nr_pieces - 1) + break; + KUNIT_EXPECT_EQ(test, + r->ar.start, start + i++ * expected_width); + KUNIT_EXPECT_EQ(test, r->ar.end, start + i * expected_width); + } + KUNIT_EXPECT_EQ(test, r->ar.start, start + i * expected_width); + KUNIT_EXPECT_EQ(test, r->ar.end, end); + damon_free_target(t); +} + +static void damon_test_split_evenly(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, damon_va_evenly_split_region(NULL, NULL, 5), + -EINVAL); + + damon_test_split_evenly_fail(test, 0, 100, 0); + damon_test_split_evenly_succ(test, 0, 100, 10); + damon_test_split_evenly_succ(test, 5, 59, 5); + damon_test_split_evenly_fail(test, 5, 6, 2); +} + +static struct kunit_case damon_test_cases[] = { + KUNIT_CASE(damon_test_three_regions_in_vmas), + KUNIT_CASE(damon_test_apply_three_regions1), + KUNIT_CASE(damon_test_apply_three_regions2), + KUNIT_CASE(damon_test_apply_three_regions3), + KUNIT_CASE(damon_test_apply_three_regions4), + KUNIT_CASE(damon_test_split_evenly), + {}, +}; + +static struct kunit_suite damon_test_suite = { + .name = "damon-primitives", + .test_cases = damon_test_cases, +}; +kunit_test_suite(damon_test_suite); + +#endif /* _DAMON_VADDR_TEST_H */ + +#endif /* CONFIG_DAMON_VADDR_KUNIT_TEST */ diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c new file mode 100644 index 000000000000..89b6468da2b9 --- /dev/null +++ b/mm/damon/vaddr.c @@ -0,0 +1,761 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DAMON Primitives for Virtual Address Spaces + * + * Author: SeongJae Park + */ + +#define pr_fmt(fmt) "damon-va: " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include "prmtv-common.h" + +#ifdef CONFIG_DAMON_VADDR_KUNIT_TEST +#undef DAMON_MIN_REGION +#define DAMON_MIN_REGION 1 +#endif + +/* + * 't->id' should be the pointer to the relevant 'struct pid' having reference + * count. Caller must put the returned task, unless it is NULL. + */ +static inline struct task_struct *damon_get_task_struct(struct damon_target *t) +{ + return get_pid_task((struct pid *)t->id, PIDTYPE_PID); +} + +/* + * Get the mm_struct of the given target + * + * Caller _must_ put the mm_struct after use, unless it is NULL. + * + * Returns the mm_struct of the target on success, NULL on failure + */ +static struct mm_struct *damon_get_mm(struct damon_target *t) +{ + struct task_struct *task; + struct mm_struct *mm; + + task = damon_get_task_struct(t); + if (!task) + return NULL; + + mm = get_task_mm(task); + put_task_struct(task); + return mm; +} + +/* + * Functions for the initial monitoring target regions construction + */ + +/* + * Size-evenly split a region into 'nr_pieces' small regions + * + * Returns 0 on success, or negative error code otherwise. + */ +static int damon_va_evenly_split_region(struct damon_target *t, + struct damon_region *r, unsigned int nr_pieces) +{ + unsigned long sz_orig, sz_piece, orig_end; + struct damon_region *n = NULL, *next; + unsigned long start; + + if (!r || !nr_pieces) + return -EINVAL; + + orig_end = r->ar.end; + sz_orig = r->ar.end - r->ar.start; + sz_piece = ALIGN_DOWN(sz_orig / nr_pieces, DAMON_MIN_REGION); + + if (!sz_piece) + return -EINVAL; + + r->ar.end = r->ar.start + sz_piece; + next = damon_next_region(r); + for (start = r->ar.end; start + sz_piece <= orig_end; + start += sz_piece) { + n = damon_new_region(start, start + sz_piece); + if (!n) + return -ENOMEM; + damon_insert_region(n, r, next, t); + r = n; + } + /* complement last region for possible rounding error */ + if (n) + n->ar.end = orig_end; + + return 0; +} + +static unsigned long sz_range(struct damon_addr_range *r) +{ + return r->end - r->start; +} + +/* + * Find three regions separated by two biggest unmapped regions + * + * vma the head vma of the target address space + * regions an array of three address ranges that results will be saved + * + * This function receives an address space and finds three regions in it which + * separated by the two biggest unmapped regions in the space. Please refer to + * below comments of '__damon_va_init_regions()' function to know why this is + * necessary. + * + * Returns 0 if success, or negative error code otherwise. + */ +static int __damon_va_three_regions(struct vm_area_struct *vma, + struct damon_addr_range regions[3]) +{ + struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0}; + struct vm_area_struct *last_vma = NULL; + unsigned long start = 0; + struct rb_root rbroot; + + /* Find two biggest gaps so that first_gap > second_gap > others */ + for (; vma; vma = vma->vm_next) { + if (!last_vma) { + start = vma->vm_start; + goto next; + } + + if (vma->rb_subtree_gap <= sz_range(&second_gap)) { + rbroot.rb_node = &vma->vm_rb; + vma = rb_entry(rb_last(&rbroot), + struct vm_area_struct, vm_rb); + goto next; + } + + gap.start = last_vma->vm_end; + gap.end = vma->vm_start; + if (sz_range(&gap) > sz_range(&second_gap)) { + swap(gap, second_gap); + if (sz_range(&second_gap) > sz_range(&first_gap)) + swap(second_gap, first_gap); + } +next: + last_vma = vma; + } + + if (!sz_range(&second_gap) || !sz_range(&first_gap)) + return -EINVAL; + + /* Sort the two biggest gaps by address */ + if (first_gap.start > second_gap.start) + swap(first_gap, second_gap); + + /* Store the result */ + regions[0].start = ALIGN(start, DAMON_MIN_REGION); + regions[0].end = ALIGN(first_gap.start, DAMON_MIN_REGION); + regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION); + regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION); + regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION); + regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION); + + return 0; +} + +/* + * Get the three regions in the given target (task) + * + * Returns 0 on success, negative error code otherwise. + */ +static int damon_va_three_regions(struct damon_target *t, + struct damon_addr_range regions[3]) +{ + struct mm_struct *mm; + int rc; + + mm = damon_get_mm(t); + if (!mm) + return -EINVAL; + + mmap_read_lock(mm); + rc = __damon_va_three_regions(mm->mmap, regions); + mmap_read_unlock(mm); + + mmput(mm); + return rc; +} + +/* + * Initialize the monitoring target regions for the given target (task) + * + * t the given target + * + * Because only a number of small portions of the entire address space + * is actually mapped to the memory and accessed, monitoring the unmapped + * regions is wasteful. That said, because we can deal with small noises, + * tracking every mapping is not strictly required but could even incur a high + * overhead if the mapping frequently changes or the number of mappings is + * high. The adaptive regions adjustment mechanism will further help to deal + * with the noise by simply identifying the unmapped areas as a region that + * has no access. Moreover, applying the real mappings that would have many + * unmapped areas inside will make the adaptive mechanism quite complex. That + * said, too huge unmapped areas inside the monitoring target should be removed + * to not take the time for the adaptive mechanism. + * + * For the reason, we convert the complex mappings to three distinct regions + * that cover every mapped area of the address space. Also the two gaps + * between the three regions are the two biggest unmapped areas in the given + * address space. In detail, this function first identifies the start and the + * end of the mappings and the two biggest unmapped areas of the address space. + * Then, it constructs the three regions as below: + * + * [mappings[0]->start, big_two_unmapped_areas[0]->start) + * [big_two_unmapped_areas[0]->end, big_two_unmapped_areas[1]->start) + * [big_two_unmapped_areas[1]->end, mappings[nr_mappings - 1]->end) + * + * As usual memory map of processes is as below, the gap between the heap and + * the uppermost mmap()-ed region, and the gap between the lowermost mmap()-ed + * region and the stack will be two biggest unmapped regions. Because these + * gaps are exceptionally huge areas in usual address space, excluding these + * two biggest unmapped regions will be sufficient to make a trade-off. + * + * + * + * + * (other mmap()-ed regions and small unmapped regions) + * + * + * + */ +static void __damon_va_init_regions(struct damon_ctx *ctx, + struct damon_target *t) +{ + struct damon_target *ti; + struct damon_region *r; + struct damon_addr_range regions[3]; + unsigned long sz = 0, nr_pieces; + int i, tidx = 0; + + if (damon_va_three_regions(t, regions)) { + damon_for_each_target(ti, ctx) { + if (ti == t) + break; + tidx++; + } + pr_debug("Failed to get three regions of %dth target\n", tidx); + return; + } + + for (i = 0; i < 3; i++) + sz += regions[i].end - regions[i].start; + if (ctx->min_nr_regions) + sz /= ctx->min_nr_regions; + if (sz < DAMON_MIN_REGION) + sz = DAMON_MIN_REGION; + + /* Set the initial three regions of the target */ + for (i = 0; i < 3; i++) { + r = damon_new_region(regions[i].start, regions[i].end); + if (!r) { + pr_err("%d'th init region creation failed\n", i); + return; + } + damon_add_region(r, t); + + nr_pieces = (regions[i].end - regions[i].start) / sz; + damon_va_evenly_split_region(t, r, nr_pieces); + } +} + +/* Initialize '->regions_list' of every target (task) */ +static void damon_va_init(struct damon_ctx *ctx) +{ + struct damon_target *t; + + damon_for_each_target(t, ctx) { + /* the user may set the target regions as they want */ + if (!damon_nr_regions(t)) + __damon_va_init_regions(ctx, t); + } +} + +/* + * Functions for the dynamic monitoring target regions update + */ + +/* + * Check whether a region is intersecting an address range + * + * Returns true if it is. + */ +static bool damon_intersect(struct damon_region *r, + struct damon_addr_range *re) +{ + return !(r->ar.end <= re->start || re->end <= r->ar.start); +} + +/* + * Update damon regions for the three big regions of the given target + * + * t the given target + * bregions the three big regions of the target + */ +static void damon_va_apply_three_regions(struct damon_target *t, + struct damon_addr_range bregions[3]) +{ + struct damon_region *r, *next; + unsigned int i; + + /* Remove regions which are not in the three big regions now */ + damon_for_each_region_safe(r, next, t) { + for (i = 0; i < 3; i++) { + if (damon_intersect(r, &bregions[i])) + break; + } + if (i == 3) + damon_destroy_region(r, t); + } + + /* Adjust intersecting regions to fit with the three big regions */ + for (i = 0; i < 3; i++) { + struct damon_region *first = NULL, *last; + struct damon_region *newr; + struct damon_addr_range *br; + + br = &bregions[i]; + /* Get the first and last regions which intersects with br */ + damon_for_each_region(r, t) { + if (damon_intersect(r, br)) { + if (!first) + first = r; + last = r; + } + if (r->ar.start >= br->end) + break; + } + if (!first) { + /* no damon_region intersects with this big region */ + newr = damon_new_region( + ALIGN_DOWN(br->start, + DAMON_MIN_REGION), + ALIGN(br->end, DAMON_MIN_REGION)); + if (!newr) + continue; + damon_insert_region(newr, damon_prev_region(r), r, t); + } else { + first->ar.start = ALIGN_DOWN(br->start, + DAMON_MIN_REGION); + last->ar.end = ALIGN(br->end, DAMON_MIN_REGION); + } + } +} + +/* + * Update regions for current memory mappings + */ +static void damon_va_update(struct damon_ctx *ctx) +{ + struct damon_addr_range three_regions[3]; + struct damon_target *t; + + damon_for_each_target(t, ctx) { + if (damon_va_three_regions(t, three_regions)) + continue; + damon_va_apply_three_regions(t, three_regions); + } +} + +static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t *pte; + spinlock_t *ptl; + + if (pmd_huge(*pmd)) { + ptl = pmd_lock(walk->mm, pmd); + if (pmd_huge(*pmd)) { + damon_pmdp_mkold(pmd, walk->mm, addr); + spin_unlock(ptl); + return 0; + } + spin_unlock(ptl); + } + + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + return 0; + pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte_present(*pte)) + goto out; + damon_ptep_mkold(pte, walk->mm, addr); +out: + pte_unmap_unlock(pte, ptl); + return 0; +} + +#ifdef CONFIG_HUGETLB_PAGE +static void damon_hugetlb_mkold(pte_t *pte, struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + bool referenced = false; + pte_t entry = huge_ptep_get(pte); + struct page *page = pte_page(entry); + + if (!page) + return; + + get_page(page); + + if (pte_young(entry)) { + referenced = true; + entry = pte_mkold(entry); + huge_ptep_set_access_flags(vma, addr, pte, entry, + vma->vm_flags & VM_WRITE); + } + +#ifdef CONFIG_MMU_NOTIFIER + if (mmu_notifier_clear_young(mm, addr, + addr + huge_page_size(hstate_vma(vma)))) + referenced = true; +#endif /* CONFIG_MMU_NOTIFIER */ + + if (referenced) + set_page_young(page); + + set_page_idle(page); + put_page(page); +} + +static int damon_mkold_hugetlb_entry(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct hstate *h = hstate_vma(walk->vma); + spinlock_t *ptl; + pte_t entry; + + ptl = huge_pte_lock(h, walk->mm, pte); + entry = huge_ptep_get(pte); + if (!pte_present(entry)) + goto out; + + damon_hugetlb_mkold(pte, walk->mm, walk->vma, addr); + +out: + spin_unlock(ptl); + return 0; +} +#else +#define damon_mkold_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ + +static const struct mm_walk_ops damon_mkold_ops = { + .pmd_entry = damon_mkold_pmd_entry, + .hugetlb_entry = damon_mkold_hugetlb_entry, +}; + +static void damon_va_mkold(struct mm_struct *mm, unsigned long addr) +{ + mmap_read_lock(mm); + walk_page_range(mm, addr, addr + 1, &damon_mkold_ops, NULL); + mmap_read_unlock(mm); +} + +/* + * Functions for the access checking of the regions + */ + +static void __damon_va_prepare_access_check(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r) +{ + r->sampling_addr = damon_rand(r->ar.start, r->ar.end); + + damon_va_mkold(mm, r->sampling_addr); +} + +static void damon_va_prepare_access_checks(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + damon_for_each_region(r, t) + __damon_va_prepare_access_check(ctx, mm, r); + mmput(mm); + } +} + +struct damon_young_walk_private { + unsigned long *page_sz; + bool young; +}; + +static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, struct mm_walk *walk) +{ + pte_t *pte; + spinlock_t *ptl; + struct page *page; + struct damon_young_walk_private *priv = walk->private; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (pmd_huge(*pmd)) { + ptl = pmd_lock(walk->mm, pmd); + if (!pmd_huge(*pmd)) { + spin_unlock(ptl); + goto regular_page; + } + page = damon_get_page(pmd_pfn(*pmd)); + if (!page) + goto huge_out; + if (pmd_young(*pmd) || !page_is_idle(page) || + mmu_notifier_test_young(walk->mm, + addr)) { + *priv->page_sz = ((1UL) << HPAGE_PMD_SHIFT); + priv->young = true; + } + put_page(page); +huge_out: + spin_unlock(ptl); + return 0; + } + +regular_page: +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + + if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) + return -EINVAL; + pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte_present(*pte)) + goto out; + page = damon_get_page(pte_pfn(*pte)); + if (!page) + goto out; + if (pte_young(*pte) || !page_is_idle(page) || + mmu_notifier_test_young(walk->mm, addr)) { + *priv->page_sz = PAGE_SIZE; + priv->young = true; + } + put_page(page); +out: + pte_unmap_unlock(pte, ptl); + return 0; +} + +#ifdef CONFIG_HUGETLB_PAGE +static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct damon_young_walk_private *priv = walk->private; + struct hstate *h = hstate_vma(walk->vma); + struct page *page; + spinlock_t *ptl; + pte_t entry; + + ptl = huge_pte_lock(h, walk->mm, pte); + entry = huge_ptep_get(pte); + if (!pte_present(entry)) + goto out; + + page = pte_page(entry); + if (!page) + goto out; + + get_page(page); + + if (pte_young(entry) || !page_is_idle(page) || + mmu_notifier_test_young(walk->mm, addr)) { + *priv->page_sz = huge_page_size(h); + priv->young = true; + } + + put_page(page); + +out: + spin_unlock(ptl); + return 0; +} +#else +#define damon_young_hugetlb_entry NULL +#endif /* CONFIG_HUGETLB_PAGE */ + +static const struct mm_walk_ops damon_young_ops = { + .pmd_entry = damon_young_pmd_entry, + .hugetlb_entry = damon_young_hugetlb_entry, +}; + +static bool damon_va_young(struct mm_struct *mm, unsigned long addr, + unsigned long *page_sz) +{ + struct damon_young_walk_private arg = { + .page_sz = page_sz, + .young = false, + }; + + mmap_read_lock(mm); + walk_page_range(mm, addr, addr + 1, &damon_young_ops, &arg); + mmap_read_unlock(mm); + return arg.young; +} + +/* + * Check whether the region was accessed after the last preparation + * + * mm 'mm_struct' for the given virtual address space + * r the region to be checked + */ +static void __damon_va_check_access(struct damon_ctx *ctx, + struct mm_struct *mm, struct damon_region *r) +{ + static struct mm_struct *last_mm; + static unsigned long last_addr; + static unsigned long last_page_sz = PAGE_SIZE; + static bool last_accessed; + + /* If the region is in the last checked page, reuse the result */ + if (mm == last_mm && (ALIGN_DOWN(last_addr, last_page_sz) == + ALIGN_DOWN(r->sampling_addr, last_page_sz))) { + if (last_accessed) + r->nr_accesses++; + return; + } + + last_accessed = damon_va_young(mm, r->sampling_addr, &last_page_sz); + if (last_accessed) + r->nr_accesses++; + + last_mm = mm; + last_addr = r->sampling_addr; +} + +static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + unsigned int max_nr_accesses = 0; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + damon_for_each_region(r, t) { + __damon_va_check_access(ctx, mm, r); + max_nr_accesses = max(r->nr_accesses, max_nr_accesses); + } + mmput(mm); + } + + return max_nr_accesses; +} + +/* + * Functions for the target validity check and cleanup + */ + +bool damon_va_target_valid(void *target) +{ + struct damon_target *t = target; + struct task_struct *task; + + task = damon_get_task_struct(t); + if (task) { + put_task_struct(task); + return true; + } + + return false; +} + +#ifndef CONFIG_ADVISE_SYSCALLS +static unsigned long damos_madvise(struct damon_target *target, + struct damon_region *r, int behavior) +{ + return 0; +} +#else +static unsigned long damos_madvise(struct damon_target *target, + struct damon_region *r, int behavior) +{ + struct mm_struct *mm; + unsigned long start = PAGE_ALIGN(r->ar.start); + unsigned long len = PAGE_ALIGN(r->ar.end - r->ar.start); + unsigned long applied; + + mm = damon_get_mm(target); + if (!mm) + return 0; + + applied = do_madvise(mm, start, len, behavior) ? 0 : len; + mmput(mm); + + return applied; +} +#endif /* CONFIG_ADVISE_SYSCALLS */ + +static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) +{ + int madv_action; + + switch (scheme->action) { + case DAMOS_WILLNEED: + madv_action = MADV_WILLNEED; + break; + case DAMOS_COLD: + madv_action = MADV_COLD; + break; + case DAMOS_PAGEOUT: + madv_action = MADV_PAGEOUT; + break; + case DAMOS_HUGEPAGE: + madv_action = MADV_HUGEPAGE; + break; + case DAMOS_NOHUGEPAGE: + madv_action = MADV_NOHUGEPAGE; + break; + case DAMOS_STAT: + return 0; + default: + return 0; + } + + return damos_madvise(t, r, madv_action); +} + +static int damon_va_scheme_score(struct damon_ctx *context, + struct damon_target *t, struct damon_region *r, + struct damos *scheme) +{ + + switch (scheme->action) { + case DAMOS_PAGEOUT: + return damon_pageout_score(context, r, scheme); + default: + break; + } + + return DAMOS_MAX_SCORE; +} + +void damon_va_set_primitives(struct damon_ctx *ctx) +{ + ctx->primitive.init = damon_va_init; + ctx->primitive.update = damon_va_update; + ctx->primitive.prepare_access_checks = damon_va_prepare_access_checks; + ctx->primitive.check_accesses = damon_va_check_accesses; + ctx->primitive.reset_aggregated = NULL; + ctx->primitive.target_valid = damon_va_target_valid; + ctx->primitive.cleanup = NULL; + ctx->primitive.apply_scheme = damon_va_apply_scheme; + ctx->primitive.get_scheme_score = damon_va_scheme_score; +} + +#include "vaddr-test.h" diff --git a/mm/page_ext.c b/mm/page_ext.c index 87df7c36dd07..dde9f33f8a2d 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -58,11 +58,21 @@ * can utilize this callback to initialize the state of it correctly. */ +#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) +static bool need_page_idle(void) +{ + return true; +} +struct page_ext_operations page_idle_ops = { + .need = need_page_idle, +}; +#endif + static struct page_ext_operations *page_ext_ops[] = { #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif -#if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT) +#if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) &page_idle_ops, #endif #ifdef CONFIG_PAGE_PINNER diff --git a/mm/page_idle.c b/mm/page_idle.c index 057c61df12db..144fb4ed961d 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -211,16 +211,6 @@ static const struct attribute_group page_idle_attr_group = { .name = "page_idle", }; -#ifndef CONFIG_64BIT -static bool need_page_idle(void) -{ - return true; -} -struct page_ext_operations page_idle_ops = { - .need = need_page_idle, -}; -#endif - static int __init page_idle_init(void) { int err; diff --git a/mm/readahead.c b/mm/readahead.c index a6bfa987a04a..a95364c99487 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -459,6 +459,8 @@ static void ondemand_readahead(struct readahead_control *ractl, if (req_size > max_pages && bdi->io_pages > max_pages) max_pages = min(req_size, bdi->io_pages); + trace_android_vh_ra_tuning_max_page(ractl, &max_pages); + /* * start of file */ diff --git a/mm/vmscan.c b/mm/vmscan.c index 89beec62bfa2..0c47c99b4cdb 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2396,6 +2396,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, denominator = ap + fp; out: trace_android_vh_tune_scan_type((char *)(&scan_balance)); + trace_android_vh_tune_memcg_scan_type(memcg, (char *)(&scan_balance)); for_each_evictable_lru(lru) { int file = is_file_lru(lru); unsigned long lruvec_size; diff --git a/net/ipv6/xfrm6_output.c b/net/ipv6/xfrm6_output.c index 6abb45a67199..ee349c243878 100644 --- a/net/ipv6/xfrm6_output.c +++ b/net/ipv6/xfrm6_output.c @@ -52,6 +52,19 @@ static int __xfrm6_output_finish(struct net *net, struct sock *sk, struct sk_buf return xfrm_output(sk, skb); } +static int xfrm6_noneed_fragment(struct sk_buff *skb) +{ + struct frag_hdr *fh; + u8 prevhdr = ipv6_hdr(skb)->nexthdr; + + if (prevhdr != NEXTHDR_FRAGMENT) + return 0; + fh = (struct frag_hdr *)(skb->data + sizeof(struct ipv6hdr)); + if (fh->nexthdr == NEXTHDR_ESP || fh->nexthdr == NEXTHDR_AUTH) + return 1; + return 0; +} + static int __xfrm6_output(struct net *net, struct sock *sk, struct sk_buff *skb) { struct dst_entry *dst = skb_dst(skb); @@ -80,6 +93,9 @@ static int __xfrm6_output(struct net *net, struct sock *sk, struct sk_buff *skb) xfrm6_local_rxpmtu(skb, mtu); kfree_skb(skb); return -EMSGSIZE; + } else if (toobig && xfrm6_noneed_fragment(skb)) { + skb->ignore_df = 1; + goto skip_frag; } else if (!skb->ignore_df && toobig && skb->sk) { xfrm_local_error(skb, mtu); kfree_skb(skb); diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index a31334b92be7..d0c95d7dd292 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2278,8 +2278,11 @@ static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, copy_skb = skb_get(skb); skb_head = skb->data; } - if (copy_skb) + if (copy_skb) { + memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0, + sizeof(PACKET_SKB_CB(copy_skb)->sa.ll)); skb_set_owner_r(copy_skb, sk); + } } snaplen = po->rx_ring.frame_size - macoff; if ((int)snaplen < 0) { @@ -3434,6 +3437,8 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, sock_recv_ts_and_drops(msg, sk, skb); if (msg->msg_name) { + const size_t max_len = min(sizeof(skb->cb), + sizeof(struct sockaddr_storage)); int copy_len; /* If the address length field is there to be filled @@ -3456,6 +3461,10 @@ static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, msg->msg_namelen = sizeof(struct sockaddr_ll); } } + if (WARN_ON_ONCE(copy_len > max_len)) { + copy_len = max_len; + msg->msg_namelen = copy_len; + } memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len); } diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 0890b42153f5..9baca999e533 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -475,7 +475,8 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { .len = IEEE80211_MAX_MESH_ID_LEN }, [NL80211_ATTR_MPATH_NEXT_HOP] = NLA_POLICY_ETH_ADDR_COMPAT, - [NL80211_ATTR_REG_ALPHA2] = { .type = NLA_STRING, .len = 2 }, + /* allow 3 for NUL-termination, we used to declare this NLA_STRING */ + [NL80211_ATTR_REG_ALPHA2] = NLA_POLICY_RANGE(NLA_BINARY, 2, 3), [NL80211_ATTR_REG_RULES] = { .type = NLA_NESTED }, [NL80211_ATTR_BSS_CTS_PROT] = { .type = NLA_U8 }, diff --git a/net/xfrm/xfrm_interface.c b/net/xfrm/xfrm_interface.c index e1fae61a5bb9..aa539f8de402 100644 --- a/net/xfrm/xfrm_interface.c +++ b/net/xfrm/xfrm_interface.c @@ -303,7 +303,10 @@ xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl) if (mtu < IPV6_MIN_MTU) mtu = IPV6_MIN_MTU; - icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); + if (skb->len > 1280) + icmpv6_ndo_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); + else + goto xmit; } else { if (!(ip_hdr(skb)->frag_off & htons(IP_DF))) goto xmit; diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile new file mode 100644 index 000000000000..8a3f2cd9fec0 --- /dev/null +++ b/tools/testing/selftests/damon/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for damon selftests + +TEST_FILES = _chk_dependency.sh +TEST_PROGS = debugfs_attrs.sh + +include ../lib.mk diff --git a/tools/testing/selftests/damon/_chk_dependency.sh b/tools/testing/selftests/damon/_chk_dependency.sh new file mode 100644 index 000000000000..0189db81550b --- /dev/null +++ b/tools/testing/selftests/damon/_chk_dependency.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 + +DBGFS=/sys/kernel/debug/damon + +if [ $EUID -ne 0 ]; +then + echo "Run as root" + exit $ksft_skip +fi + +if [ ! -d "$DBGFS" ] +then + echo "$DBGFS not found" + exit $ksft_skip +fi + +for f in attrs target_ids monitor_on +do + if [ ! -f "$DBGFS/$f" ] + then + echo "$f not found" + exit 1 + fi +done diff --git a/tools/testing/selftests/damon/debugfs_attrs.sh b/tools/testing/selftests/damon/debugfs_attrs.sh new file mode 100644 index 000000000000..196b6640bf37 --- /dev/null +++ b/tools/testing/selftests/damon/debugfs_attrs.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +test_write_result() { + file=$1 + content=$2 + orig_content=$3 + expect_reason=$4 + expected=$5 + + echo "$content" > "$file" + if [ $? -ne "$expected" ] + then + echo "writing $content to $file doesn't return $expected" + echo "expected because: $expect_reason" + echo "$orig_content" > "$file" + exit 1 + fi +} + +test_write_succ() { + test_write_result "$1" "$2" "$3" "$4" 0 +} + +test_write_fail() { + test_write_result "$1" "$2" "$3" "$4" 1 +} + +test_content() { + file=$1 + orig_content=$2 + expected=$3 + expect_reason=$4 + + content=$(cat "$file") + if [ "$content" != "$expected" ] + then + echo "reading $file expected $expected but $content" + echo "expected because: $expect_reason" + echo "$orig_content" > "$file" + exit 1 + fi +} + +source ./_chk_dependency.sh + +# Test attrs file +# =============== + +file="$DBGFS/attrs" +orig_content=$(cat "$file") + +test_write_succ "$file" "1 2 3 4 5" "$orig_content" "valid input" +test_write_fail "$file" "1 2 3 4" "$orig_content" "no enough fields" +test_write_fail "$file" "1 2 3 5 4" "$orig_content" \ + "min_nr_regions > max_nr_regions" +test_content "$file" "$orig_content" "1 2 3 4 5" "successfully written" +echo "$orig_content" > "$file" + +# Test schemes file +# ================= + +file="$DBGFS/schemes" +orig_content=$(cat "$file") + +test_write_succ "$file" "1 2 3 4 5 6 4 0 0 0 1 2 3 1 100 3 2 1" \ + "$orig_content" "valid input" +test_write_fail "$file" "1 2 +3 4 5 6 3 0 0 0 1 2 3 1 100 3 2 1" "$orig_content" "multi lines" +test_write_succ "$file" "" "$orig_content" "disabling" +echo "$orig_content" > "$file" + +# Test target_ids file +# ==================== + +file="$DBGFS/target_ids" +orig_content=$(cat "$file") + +test_write_succ "$file" "1 2 3 4" "$orig_content" "valid input" +test_write_succ "$file" "1 2 abc 4" "$orig_content" "still valid input" +test_content "$file" "$orig_content" "1 2" "non-integer was there" +test_write_succ "$file" "abc 2 3" "$orig_content" "the file allows wrong input" +test_content "$file" "$orig_content" "" "wrong input written" +test_write_succ "$file" "" "$orig_content" "empty input" +test_content "$file" "$orig_content" "" "empty input written" +echo "$orig_content" > "$file" + +echo "PASS" diff --git a/tools/testing/selftests/filesystems/incfs/Makefile b/tools/testing/selftests/filesystems/incfs/Makefile index f3798029247a..5a2f6301c4b2 100644 --- a/tools/testing/selftests/filesystems/incfs/Makefile +++ b/tools/testing/selftests/filesystems/incfs/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -CFLAGS += -D_FILE_OFFSET_BITS=64 -Wall -Werror -I../.. -I../../../../.. -LDLIBS := -llz4 -lzstd -lcrypto -lpthread +CFLAGS += -D_FILE_OFFSET_BITS=64 -Wall -Werror -I../.. -I../../../../.. -fno-omit-frame-pointer -fsanitize=address -g +LDLIBS := -llz4 -lzstd -lcrypto -lpthread -fsanitize=address TEST_GEN_PROGS := incfs_test incfs_stress incfs_perf include ../../lib.mk diff --git a/tools/testing/selftests/filesystems/incfs/incfs_test.c b/tools/testing/selftests/filesystems/incfs/incfs_test.c index 2d2bc933fefc..bfd5f1ac7c7a 100644 --- a/tools/testing/selftests/filesystems/incfs/incfs_test.c +++ b/tools/testing/selftests/filesystems/incfs/incfs_test.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -43,6 +44,7 @@ #define FS_IOC_GETFLAGS _IOR('f', 1, long) #define FS_VERITY_FL 0x00100000 /* Verity protected inode */ +#define TEST_SKIP 2 #define TEST_FAILURE 1 #define TEST_SUCCESS 0 @@ -2793,14 +2795,16 @@ static int get_hash_blocks_test(const char *mount_dir) return TEST_FAILURE; } +#define THREE_GB (3LL * 1024 * 1024 * 1024) +#define FOUR_GB (4LL * 1024 * 1024 * 1024) /* Have 1GB of margin */ static int large_file_test(const char *mount_dir) { char *backing_dir; int cmd_fd = -1; int i; - int result = TEST_FAILURE; + int result = TEST_FAILURE, ret; uint8_t data[INCFS_DATA_FILE_BLOCK_SIZE] = {}; - int block_count = 3LL * 1024 * 1024 * 1024 / INCFS_DATA_FILE_BLOCK_SIZE; + int block_count = THREE_GB / INCFS_DATA_FILE_BLOCK_SIZE; struct incfs_fill_block *block_buf = calloc(block_count, sizeof(struct incfs_fill_block)); struct incfs_fill_blocks fill_blocks = { @@ -2809,6 +2813,22 @@ static int large_file_test(const char *mount_dir) }; incfs_uuid_t id; int fd = -1; + struct statvfs svfs; + unsigned long long free_disksz; + + ret = statvfs(mount_dir, &svfs); + if (ret) { + ksft_print_msg("Can't get disk size. Skipping %s...\n", __func__); + return TEST_SKIP; + } + + free_disksz = (unsigned long long)svfs.f_bavail * svfs.f_bsize; + + if (FOUR_GB > free_disksz) { + ksft_print_msg("Not enough free disk space (%lldMB). Skipping %s...\n", + free_disksz >> 20, __func__); + return TEST_SKIP; + } backing_dir = create_backing_dir(mount_dir); if (!backing_dir) @@ -2849,6 +2869,7 @@ static int large_file_test(const char *mount_dir) failure: close(fd); close(cmd_fd); + unlink("very_large_file"); umount(mount_dir); free(backing_dir); return result; @@ -4665,9 +4686,15 @@ struct test_case { void run_one_test(const char *mount_dir, struct test_case *test_case) { + int ret; + ksft_print_msg("Running %s\n", test_case->name); - if (test_case->pfunc(mount_dir) == TEST_SUCCESS) + ret = test_case->pfunc(mount_dir); + + if (ret == TEST_SUCCESS) ksft_test_result_pass("%s\n", test_case->name); + else if (ret == TEST_SKIP) + ksft_test_result_skip("%s\n", test_case->name); else ksft_test_result_fail("%s\n", test_case->name); } diff --git a/tools/testing/selftests/filesystems/incfs/utils.c b/tools/testing/selftests/filesystems/incfs/utils.c index e60b0d36f49d..0411d912ae05 100644 --- a/tools/testing/selftests/filesystems/incfs/utils.c +++ b/tools/testing/selftests/filesystems/incfs/utils.c @@ -116,6 +116,9 @@ size_t format_signature(void **buf, const char *root_hash, const char *add_data) size_t size = sizeof(struct signature_blob) + strlen(add_data) + 1; struct signature_blob *sb = malloc(size); + if (!sb) + return 0; + *sb = (struct signature_blob){ .version = INCFS_SIGNATURE_VERSION, .hash_section_size = sizeof(struct hash_section), @@ -126,7 +129,7 @@ size_t format_signature(void **buf, const char *root_hash, const char *add_data) .salt_size = 0, .hash_size = SHA256_DIGEST_SIZE, }, - .signing_section_size = sizeof(uint32_t) + strlen(add_data) + 1, + .signing_section_size = strlen(add_data) + 1, }; memcpy(sb->hash_section.hash, root_hash, SHA256_DIGEST_SIZE);