From ca09ddd123bbb57a1636f243f2d28e23e8d2f103 Mon Sep 17 00:00:00 2001 From: Liubov Talamanova Date: Tue, 1 Feb 2022 15:15:20 +0300 Subject: [PATCH] [POT] Implement DataFreeEngine (#9484) * [POT] Implement DataFreeEngine * Add CLI * Updated CLI * Moved logic to SynteticImageLoader * Fix bug with draw modes * Fix bug in DataFreeEngine * Fix multiprocessing * Fix pylint * Add DataFreeEngine test * Download models * Fill background * Fix test * Fix args * Support config option for DataFree mode * Minor fixes * Add data_free config * Add more test cases * Enable RCNN models quantization --- .../pot/configs/data_free_mode_template.json | 34 ++ tools/pot/openvino/tools/pot/app/argparser.py | 44 ++- tools/pot/openvino/tools/pot/app/run.py | 10 +- .../pot/openvino/tools/pot/configs/config.py | 17 +- .../tools/pot/data_loaders/creator.py | 14 +- .../tools/pot/data_loaders/image_loader.py | 23 +- .../pot/data_loaders/synthetic_background.npy | Bin 0 -> 24128 bytes .../data_loaders/synthetic_image_loader.py | 327 ++++++++++++++++++ .../pot/openvino/tools/pot/engines/creator.py | 3 + .../tools/pot/engines/data_free_engine.py | 19 + tools/pot/tests/test_data_generation.py | 51 +++ tools/pot/tests/test_sanity.py | 18 + 12 files changed, 544 insertions(+), 16 deletions(-) create mode 100644 tools/pot/configs/data_free_mode_template.json create mode 100644 tools/pot/openvino/tools/pot/data_loaders/synthetic_background.npy create mode 100644 tools/pot/openvino/tools/pot/data_loaders/synthetic_image_loader.py create mode 100644 tools/pot/openvino/tools/pot/engines/data_free_engine.py create mode 100644 tools/pot/tests/test_data_generation.py diff --git a/tools/pot/configs/data_free_mode_template.json b/tools/pot/configs/data_free_mode_template.json new file mode 100644 index 00000000000..8c621fb7b6b --- /dev/null +++ b/tools/pot/configs/data_free_mode_template.json @@ -0,0 +1,34 @@ +{ + "model": { + "model_name": "model_name", // Model name + "model": "", // Path to model (.xml format) + "weights": "" // Path to weights (.bin format) + }, + "engine": { + + "type": "data_free", // Engine type​ + "generate_data": "True", // (Optional) If True, generate synthetic data and store to `data_source`​ + // Otherwise, the dataset from `--data-source` will be used'​ + "layout": "NCHW", // (Optional) Layout of input data. Supported: ["NCHW", "NHWC", "CHW", "CWH"]​ + "shape": "[None, None, None, None]", // (Optional) if model has dynamic shapes, input shapes must be provided​ + "data_type": "image", // (Optional) You can specify the type of data to be generated.​ + // Currently only `image` is supported.​ + // It is planned to add 'text` and 'audio' cases​ + "data_source": "PATH_TO_SOURCE" // (Optional) You can specify path to directory​ + // where synthetic dataset is located or will be generated and saved​ + }, + "compression": { + "algorithms": [ + { + "name": "DefaultQuantization", // Optimization algorithm name + "params": { + "preset": "performance", // Preset [performance, mixed, accuracy] which control the quantization + // mode (symmetric, mixed (weights symmetric and activations asymmetric) + // and fully asymmetric respectively) + "stat_subset_size": 300 // Size of subset to calculate activations statistics that can be used + // for quantization parameters calculation + } + } + ] + } +} diff --git a/tools/pot/openvino/tools/pot/app/argparser.py b/tools/pot/openvino/tools/pot/app/argparser.py index 06290bb7ac4..50a10f9916e 100644 --- a/tools/pot/openvino/tools/pot/app/argparser.py +++ b/tools/pot/openvino/tools/pot/app/argparser.py @@ -13,7 +13,8 @@ def get_common_argument_parser(): parser.add_argument( '-c', '--config', - help='Path to a config file with optimization parameters. Overrides "-q | -m | -w | --ac-config" options') + help='Path to a config file with optimization parameters. ' + 'Overrides "-q | -m | -w | --ac-config | --engine" options') parser.add_argument( '-q', @@ -47,6 +48,12 @@ def get_common_argument_parser(): type=str, help='Model name. Applicable only when -q option is used.') + parser.add_argument( + '--engine', + choices=['accuracy_checker', 'data_free', 'simplified'], + type=str, + help='Engine type. Default: `accuracy_checker`') + parser.add_argument( '--ac-config', type=str, @@ -105,6 +112,37 @@ def get_common_argument_parser(): default=False, help='Keep Convolution, Deconvolution and FullyConnected weights uncompressed') + data_free_opt = parser.add_argument_group('DataFreeEngine options') + + data_free_opt.add_argument( + '--data-source', + default='../../../pot_dataset', + help='Path to directory where synthetic dataset is located or will be generated and saved. ' + 'Default: `../../../pot_dataset`') + + data_free_opt.add_argument( + '--shape', + type=str, + help='Required for models with dynamic shapes. ' + 'Input shape that should be fed to an input node of the model. ' + 'Shape is defined as a comma-separated list of integer numbers enclosed in ' + 'parentheses or square brackets, for example [1,3,227,227] or (1,227,227,3), where ' + 'the order of dimensions depends on the framework input layout of the model.') + + data_free_opt.add_argument( + '--data-type', + type=str, + default='image', + choices=['image'], + help='Type of data for generation. Dafault: `image`') + + data_free_opt.add_argument( + '--generate-data', + action='store_true', + default=False, + help='If specified, generate synthetic data and store to `data-source`. ' + 'Otherwise, the dataset from `--data-source` will be used') + return parser @@ -112,7 +150,7 @@ def check_dependencies(args): if (args.quantize is not None and (args.model is None or args.weights is None or - args.ac_config is None)): + args.ac_config is None and args.engine != 'data_free')): raise ValueError( '--quantize option requires model, weights, and AC config to be specified.') if args.quantize is None and args.config is None: @@ -122,6 +160,8 @@ def check_dependencies(args): raise ValueError('Either --config or --quantize option should be specified') if args.quantize == 'accuracy_aware' and args.max_drop is None: raise ValueError('For AccuracyAwareQuantization --max-drop should be specified') + if args.engine == 'data_free' and args.ac_config is not None: + raise ValueError('Either DataFree mode or AC config should be specified') check_extra_arguments(args, 'model') check_extra_arguments(args, 'weights') check_extra_arguments(args, 'preset') diff --git a/tools/pot/openvino/tools/pot/app/run.py b/tools/pot/openvino/tools/pot/app/run.py index fe22f257904..0e5e6b36918 100644 --- a/tools/pot/openvino/tools/pot/app/run.py +++ b/tools/pot/openvino/tools/pot/app/run.py @@ -35,11 +35,17 @@ def app(argv): _update_config_path(args) config = Config.read_config(args.config) + + if args.engine: + config.engine['type'] = args.engine if args.engine else 'accuracy_checker' + if 'data_source' not in config.engine: + config.engine['data_source'] = args.data_source + config.configure_params(args.ac_config) config.update_from_args(args) - if config.engine.type == 'simplified' and args.evaluate: - raise Exception('Can not make evaluation in simplified mode') + if config.engine.type != 'accuracy_checker' and args.evaluate: + raise Exception('Can not make evaluation in simplified or data_free mode') log_dir = _create_log_path(config) init_logger(level=args.log_level, diff --git a/tools/pot/openvino/tools/pot/configs/config.py b/tools/pot/openvino/tools/pot/configs/config.py index 7e6ee24665c..adc2d4c214f 100644 --- a/tools/pot/openvino/tools/pot/configs/config.py +++ b/tools/pot/openvino/tools/pot/configs/config.py @@ -63,6 +63,19 @@ class Config(Dict): self.model['output_dir'] = args.output_dir self.model['direct_dump'] = args.direct_dump self.engine['evaluate'] = args.evaluate + if self.engine.type == 'data_free': + if 'data_type' not in self.engine: + self.engine['data_type'] = args.data_type + if 'generate_data' not in self.engine: + self.engine['generate_data'] = args.generate_data + if 'shape' not in self.engine: + self.engine['shape'] = args.shape + if self.engine['generate_data']: + subset_size = 0 + for algo in self.compression['algorithms']: + subset_size = max(subset_size, algo.get('stat_subset_size', 300)) + self.engine['subset_size'] = subset_size + self.model['keep_uncompressed_weights'] = args.keep_uncompressed_weights if 'optimizer' in self: self.optimizer.params['keep_uncompressed_weights'] = args.keep_uncompressed_weights @@ -295,9 +308,9 @@ class Config(Dict): if 'type' not in engine or engine.type == 'accuracy_checker': self._configure_ac_params() self.engine.type = 'accuracy_checker' - elif engine.type == 'simplified': + elif engine.type == 'simplified' or engine.type == 'data_free': if 'data_source' not in engine: - raise KeyError('Missed data dir for sample engine') + raise KeyError(f'Missed data dir for {engine.type} engine') self.engine.device = engine.device if engine.device else 'CPU' engine.data_source = Path(engine.data_source) else: diff --git a/tools/pot/openvino/tools/pot/data_loaders/creator.py b/tools/pot/openvino/tools/pot/data_loaders/creator.py index 356f3b53f18..65411f73bb9 100644 --- a/tools/pot/openvino/tools/pot/data_loaders/creator.py +++ b/tools/pot/openvino/tools/pot/data_loaders/creator.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from openvino.tools.pot.data_loaders.image_loader import ImageLoader +from openvino.tools.pot.data_loaders.synthetic_image_loader import SyntheticImageLoader from openvino.tools.pot.graph.model_utils import get_nodes_by_type @@ -24,9 +25,16 @@ def create_data_loader(config, model): data_loader = None for in_node in inputs: if tuple(in_node.shape) != (1, 3): - data_loader = ImageLoader(config) - data_loader.shape = in_node.shape - data_loader.get_layout(in_node) + if config.type == 'simplified': + data_loader = ImageLoader(config) + data_loader.shape = in_node.shape + data_loader.get_layout(in_node) + elif config.type == 'data_free': + if not config.shape: + config.shape = in_node.shape + if not config.layout: + config.layout = in_node.graph.graph.get('layout', None) + data_loader = SyntheticImageLoader(config) return data_loader if data_loader is None: diff --git a/tools/pot/openvino/tools/pot/data_loaders/image_loader.py b/tools/pot/openvino/tools/pot/data_loaders/image_loader.py index 97a37b4954d..27edcfa4880 100644 --- a/tools/pot/openvino/tools/pot/data_loaders/image_loader.py +++ b/tools/pot/openvino/tools/pot/data_loaders/image_loader.py @@ -33,25 +33,34 @@ class ImageLoader(DataLoader): self._shape = tuple(shape) def _read_and_preproc_image(self, img_path): + C = self._layout.get_index_by_name('C') + H = self._layout.get_index_by_name('H') + W = self._layout.get_index_by_name('W') + image = imread(img_path, IMREAD_GRAYSCALE)\ - if self._shape[1] == 1 else imread(img_path) + if self._shape[C] == 1 else imread(img_path) if image is None: raise Exception('Can not read the image: {}'.format(img_path)) - return prepare_image(image, self._layout, self.shape[-2:], self._crop_central_fraction) + return prepare_image(image, self._layout, (self.shape[H], self.shape[W]), self._crop_central_fraction) - def get_layout(self, input_node): + def get_layout(self, input_node=None): if self._layout is not None: if 'C' not in self._layout or 'H' not in self._layout or 'W' not in self._layout: raise ValueError('Unexpected {} layout'.format(self._layout)) + if self._shape is not None and 'N' in self._layout and len(self._shape) == 3: + self._layout = self._layout[1:] self._layout = Layout(self._layout) return - layout_from_ir = input_node.graph.graph.get('layout', None) - if layout_from_ir is not None: - self._layout = Layout(layout_from_ir) - return + if input_node: + layout_from_ir = input_node.graph.graph.get('layout', None) + if layout_from_ir is not None: + if self._shape is not None and 'N' in layout_from_ir and len(self._shape) == 3: + layout_from_ir = layout_from_ir[1:] + self._layout = Layout(layout_from_ir) + return image_colors_dim = (Dimension(3), Dimension(1)) num_dims = len(self._shape) diff --git a/tools/pot/openvino/tools/pot/data_loaders/synthetic_background.npy b/tools/pot/openvino/tools/pot/data_loaders/synthetic_background.npy new file mode 100644 index 0000000000000000000000000000000000000000..28da19dee252afceda0d0cdbc8e6b2780a967b9d GIT binary patch literal 24128 zcmbST_dnKe7bhAj*+fZ5GNK_Wl_S}kd+(c+WMq_C6e^*ij7VfBqs+34kVrx)GplGQ z5+(JV@4xW$TVB%r`CQjI=e>^WRzyo1Dlg9|on+vS$(7O5f3jP zFRnY=!|Slqe_uD^x_Ud|*S(#&C!Fx_Teff8woQHGP8Icyr#Jq8e^xf0tt)kKCqY+_ zRg;P&9ZG@rQ+(3t) zjk_1Uep>)zSu-|F@)(2@!2p##bcCU~oOGEhEqb z>Wg{iRdg7zMQf*xa0wG8dM^k3IYou}{qv5F%ehb{M5(>}hXW!ls_7hidsy6>>0Xsa zg1G~lQVP%LpqrYo=zEO~QppcO)=U$i?WUe}l^G3$v}8GHyO`i(GM$zuKnJb?=a7^F z2No^jH)lC=;IYtbzm6XRdc%+Iw3O$-8+W0kyK6Y`(3rB8q)vkOSB}UXvtz=m;VwG` zIR-e|>iH}4Goa*i#q?$;Hrz4V6wCLI4eMME-yL0`fqj;??inK%oWILsSZ~CJWJ#{f z>N`|W&Q{1DE9bx=?TIss8<@~~``DgeBQ$6np10^<#(@~mJdZ2vWL-8LGi`s~Vb>ZO8*32RLC8x5kklFLRnv*BzI zU$>Yr6Q;Ck+uK%A!MVO@GfkWc>UsZ4zrA9B{S_b0&0;hdy(&I*6Ti>tsBfAuK3}wW zipEk!M+oqLRrxr=0i15zuykFy@HyJ0R49-G>YWk9kbNxp(q6ARFGh!derL_CWmw>q zDZgDuCLP8qDWRr(J$bg>7RKO{&p0aD&(WT`rjh`zJ?lB zVM>SitjEvT(oFFB$hX*J%7T#y2ZB}*1I9V2RYQkJ(B5g|&3lOnXP1B2SAUKH>s=+3 zLLBg(T)v?vDNP1P-sx>|^>kQj7U@wf&w`&+x~^aX33zWm$<;nVhs>JpULGQzgLRF# z`35qC)Unp~J21h%)%fV(C=0H{WczpY62a3svt;lr6>|0lzK~kahV|mci6ueyFxvN9 zg+--8!ASH$fnEym39XI|$g+nbQ`;3XTUoGGr*f1L$$(c;B9}tHP~cUdZ)Dzj7P#J& zjoBf<1=A{j*QZWosO$M>5FA7Yxy_I6pGaoG>3V;f4cLS0;TO3RiBzc9@#r(a`czrn za#U@B3dzkA-CPEOD~;E&A^1T@h|oGZJ^iA&5tLO z3Ze#|zm74<@c2h?Zof1Y1eSZ2gkrr#)bamnuwlbSK7ZBN$3&QWWb3!fjt)+2!95#7 zS#ao^?0oJa8n|58T5+|U1?ysq4%`T1LrC}u&HfXXz*`_Pe7J=Oe`@ATvKiMvx&=xpLfX`xga-Ex~(7tnVpje3s{g-Q|A6~$7`F-*f zx04Ent|NOZMHpZa=9KighzM>G0R-WHn71iJZ(0i$(RF!(X%=Dt1>)_2SP zj1D3c|Ey~8ct?lO#!&Xs0v7OnNIx&ALj|wa?Isr!Nubj=A>>bH!t?S54W;v3$jIka zX|$q1&Wyr@^Z*?ycTT1~W3fPQk*;QchYT4G0Yk)<3=qQmie_R9l ztZy=bdhc3pkuL`dj#kFFzQ=Q4Exa-2Fb9_1enEYAp9OZh#OT%!EKuORCOy!K{X9D> z&4=Uw7LFp`K@ntVkrF!g)RqIfy4(k|8}PobFt7ZlNP@N5=l{BYri0Q!+Gr*n`$;_0 z&fSv?8q2PJl6b@bO6PHT{$LuMQY1E+e`mpGpMdwSksL6&)u5f?!G$$9cF82iQQ*{} zR@Xib1C+WVX}6!SVMb1s80JTZLUH~h1Z57$F@6i&--B~8^8IRHvEd~>KtJ@j1L)tS zu$>DW!Q)5nLi7w7wopEd?;hnJ)9q9H_tl$?^an7Z_Cj zx%3t~$opr?t#o0+wCCfC#MLYiDC#~e;z|RhzeCG4O<17+$?cmw_Nl$ib;HTIH2B)) z>_OPhfwxjOzO?pp;f}KCc_}=XQ_Ag2wUmmvS zz}BUg+>#^NAjGr7L6S@X+f^UGFWPf$LL9hR?>G#}t(~Fuq_U-G=jAj_>9p@?j#}syppY`ACJr5|^O2J{0H)5IU4{ z58uN%w^!JIGA2LaZ#278lQKik0&xgtg`b?RXQDvY8Flh z&9XtSU2t#6W2|e9l?Q!2>9C-+IYx8`3sRWRV`n%lINa}M!1f?Q<_}T6Tg&O-;@+!1 zp+*M{%EW`zZ8Z3)w(g8Se(uvxCTO8d4wU`wb(1h8z}Z%NqZDrn`0P(1-xH@o(aC#W zCb3j#$mCtyt;~Yy<3Gb{@!Wb|DQ6pPq`}>@RmwcZ4DiXRviRl9gso9^yvuh`AX>v{ zyZjM4T-UhQ#687;vO$vVSL}=WZ4Y*PJJ`W@p-m5Isiq*Z-6|}VK!U`*eJ^R_6sUj8 zJ=^n^1`Wq{ch_{#Kygp7llKe@l2othobn{VcjvGip(=Y|79QpwaHN1))`IlElMJ9; zhqWSN7VvCeS4)tg9lR}j=cYJf3#QhZdHYkz;Hl*Oj)-$QYnkVrurCbo-muoH{}=(z zzcJLzUqOSu_vSsz#2C;}sQ1@5n*y{wX|+q#nLtxma$wPg4U%WOxjr`!y09y(NAJO*IUvceVf|w3!LATMOU59ufsmyw>riu zhYDg>r7cfyMVuGB*E=thjkq>sG5#?D);Aj*;MJsnb?Peh2hW)BJH>$LI!J_qNW~~! zM=E^#v}CRkajuMUpxNyt5(xND$v#Q^CgrxeQ3!{>d;vhYgm&i(V`}obLrDTRO%_kRj}CA?8E{pLP@5Iu`~Ic0ASN zBeS9T;*vqN{T#5^eJ^xJI|B~;TD{~IWq@7V)MrtwtGJTm(eHd&Q2A?deVGCslH~30 z*#Pp_oXLQlsrjo)x7)+ev)ylPxny_{eoATTIvsIs=3H?K1t@(teEYXB!DavL z6#=X1@HDn@k)BI}LsC@P4`UQa6}@xd)F>HFUt^115GTRJc${(J9|kOYIC^MmfdLhF zv`2jWsqk<^uduTM3kIKP>-rQjLG*{lH#ZwHkk^EKdy4&Y$}#)MMK^pd{k;eLfeZ~5 zoi9gEvEb>#>E-A3nb67EbGma43-0g<4fMQcLxxe(rI)EJkbAJ~Ns$`%nbZ0w=k~F| zmE`I+Hq3$VuvGWQ7B&nS?ffWn3Gwg4Y}39u4rsPH#y|MQhE~VTG2aLrAgTD=s!`>@ z$90nGAucQkUeDkaH{?LbP_nVG7aQ)*Z6^xFvw?iYevyoQb>%%}=L?Pa-t`Q)k#<~= zpH@%ecjSW9gj7)Lc@A(356(KDWx?u%uJGfJSO7{I-`D^59@ehgwr>{$HgD=6Dp(=z z_RFx*iDJSbB{hX~Z6<^?(d{L^&_VxM$XWVL><50&TMyT8VDncu1Q(3a)b_ten_jvzq5yRT@j0ETzp=qE9Wl0V}qf&-S4g8W)QO^d!{jf z0dK_b-7%#wL1#|_?do$H9GO4H4r!x6MZLFG_#hn~?or-zG=dE+3Q;ZxifE8J2`_X6 z*`T<$vd&W+=hVVJdl7y*?};1U~F6wFT_7p8z-%+uL_$QSDLH;N|N zvSD{pzq=ln0$J(gh7Ia;u*fieduNORi3SM|#k|O%bn)K#8hqZh_f9oy++u=fj!UO@ z0~!9Ux4NL{!+@X>zax@t7FbI~7=7MNfw}m^OTDsq58hUN`;C2c(RF2I;a>{GbUyh| z_>BqtLLu78VmQ~>f3@@W(7;*wQFF^D7Vs#P^8PMlL%LWvH9Ct7>q)|nH&f`KrE&Vc z3Erpoawc6awG^1lSjc!}#sEu!-t5OBELgMRb;eJ`RROK)g{1;4Fn->wqHn>08yWgl zM%7f9l8OlzN8Uw}i0$dy#)hR6jae{EhJ-iUKRMyNEOfs$^{Eo;t2BnLa*+yLiETCQ zbOvY#@TxGDGr^9kJ?5 z{JFqwpNe~n^E$=gT1i?m9i#*%2F_ufx0xKcRCEG)=&mxU>C4E=S7g;E4>N&ht!qsd z;_b0Krueuf4cfAPTCLxY_X-&c>BbQ$c6M;J}|vOxXA9)9dObOc01@z9ok^ZGE3_E7;N@@saZ? zm6r^dT4YzyC#f*`$MDF$|LT3b3C(qRS^{{7>0O@{?7Wh#VhDGCJCyhvWBjrIPtdvi73!zg--!b(4UZ^8eR z2sx-52r=~{rk8=u%lE&g0`h^Hc{|z&N(DVB8(wm)NayD>fkS;=BKe ze=i4Gx?~J)-J^rdY}l5`6vTh>#cKCIu|a!@e%=wB*YE0Te+i4R;qqd(jxqN0VE0&Y zD`OhC-ffutTgQfaQ85XPEDjj$Z@eq~o(MX3pYK@moeH@{t*gCpJ{qL#o^;0huNf2i zG}xU18bzDeKh|eM3|PcxT*iJ?Q((NWo(-#oA5z$DY%uG#Uz&nnKXOXc(h&P#QRUbn ziE$>l(K23E7cxM;x=V4JJL0OD3RVt2UuE+);Z8iSM3sy7*>_kV>EB^jQjWZP&fvoL zL>gRn?jsIgWr3&Dcn2qx4dV$dLJI3CAT#50ZmR|p27f8rRn)RzbHonvy}dLzlvw?9 zx{e7WdlFibjahI#;i-6{Bn9YB*GYHdXpnX0_p?82Cgl3$EH*D?z}Z7TzF)UvK~BR_ zx!;%Zy_LrtI>@3!#^%4D)v-@r7x}X@19^egdl^FQ6a{#kHTYf;5%(YdwGlE|?7_w_E16FA41Iyr)qOxQ~(N@_7AL8w<% z;oU|WY@D^t6-N9!uPL$m`Vis@krxUzGIZ#0y3%g4nFZAYL5#8-0>mw5jfd!wVJ_8l zuI308HoqPI7lAx#>m=1i9_N&-%3Yl@3dsmmElPCpR#m<7G-< zb{7r!WL5cfo7up>pX}g_^LCB4OJPDY1(JV|qm(}}p?>MHezo1GKS^r53ph*xu?IT4 zohb}p?r-wuU&)4o)v>fTKI}WCa(9Q3mx?#=a_sUr(0>1zm;NIfWKb7JH=kz1*N*S6 z+^!^F=_f(_muf|VJrgqPj7`0U+2H*rYVzF@GDs%dhYuUlpb|bmd9alZF=0KX zv5&}*e%1Ts@IeMx|9!OS7xJf9l`iWh4^csKmtJHt&fS3J>PIEB@toy|>V2rc{2Y}2 zOfRRwzAK5pU!B3Zy=}6{$_~%v!qR-{I^@^8_?(SvQI~t}GVN!9?|+H^eiaECXjc?0 zhTu8qpJNn7E~7%#b%A8QAJ`Wj$A6`+V}aBC<2`kOWGGu;>1Pb%`94+8PI%7(O6C6K zm!~K&-M!tjyPg4t2T~>Me<5#B<1We$&|sO5RcEUJ2R8lLaitt}Je`se+ngji(4UDX zE*)S(P=aEs@Fo@repzEI-$sXLpX96m#*q;eovg~ab@bHP8 z!Mr5{MmN81{J}?sH~UqX4@6n8rL@mt26ef8K6{>CFr$Ku!}9yTN?FjhG4D&m9;}zt z+V6Q|G}!T0d~u|Q1sjtx(}rKLq5bnZiS2YMG;c3cyIRkL{ZXdU=2&MQ`E`4k*QxNk z{L13CY3%!qj_vGbCOo}fbIz}e1lm(y&Mm2-!o;IRhup_>n9eV$^)#UXr%^fRodyFg zx@`)|?O?!_h~zpR)I->FWdaedc&-;lN^OXAXd!g)2G=lvT4U<-!I}dq_seRfoH*DQ zJ6MUxQ=)d|o@vB+x#F6;8h4Nl&yHEN$vV+tbFo&Y58^dKdr~BO3u+>LciQ~djJyxm-B6!pQnMRkWG2s3MR;m zTYZ~Y&xEy3j`g>-XmF=Eaerqc>Sk5agbgB0u+@BiELVsOdsTkOoDx7hV5i^xEfzoj zZfEtcBNVvEZ_P|Yew(UO=65-X3F0FM$CVw)U@5OWqaK6!+|KgSjd&IqsC8~CTS)=+ zpLs6})-mA@Nh*I^4I831bu1oa4W6o~=#yZ7TbXI%RuO{m4Pog{Dg%7vb06 zEZG_q5yylCsqTj3LR2WGDhcPK?kf;(f5Wtz3GceZ+_H8v!1nUIbr6{aGI!6XUghOL zP}rpe!y)8-NxGB7Z8*1H6Q2&7v%sv1@XsQc0PvCVWp;-N7FGNi;#D)D<#;g!TT(_3Y9ICZX&LbJf<>a>_3m@r_F zcQRb|CmqzoY<14@lHln&H;YrK!|UW+A=Cx3pnt{A`>U={;ks`6b48r@0m7QC&k9(m z_htn0l#pQE%aVQf)EKaO2X`wGaf+C{V#T644f0$pe>OBQ;M9dQdL+)H{W3>#7Oqhs zNKucv8hL4-uJga9eg;g8HkLVmq=V^#Y2tRg{||3I>fBz#0zd7zE%S&&f99-CP$*@> z@-*JId1KVg{J!|t$B@CtOjYYE>h>39{|f2MA#O2$Tov&jKU+~5q+i2?)goixh=>zk ztX7px`G9k(W8K`sQ5J;zXZBY_aln1k3x8i*DmeYSB-(@b$MD$Z2e1R47C6O{O{pSz~{u!^)2QSBlLv zFv#FhVsp@cm^mb?iaOw%tuF>NMcClX__d^-pADmfA%yrL8VIiPkgCwZbB({hs5`-e z{oYgK$FN^T)xMjP?4v>ad%m+IWd@`eM_YeSpu*DwrE&tl5pP_TjnVo}2RGrbD~{!o zp`baLp?RJO&BZ?x4~lTWY}xk4nFc!4Q$(y!GuWWz|1kDNGzYBSDW!y@6CoqP_c0f7 zWmPk^Z?^*8!>#n3MFs&plD!vVwxKS{(*G*Gi2OSEmgcQ4GK?CTI)(|cpnlik0u`Ls z6t(ar_fXGyyrz)%D9(xF*Vi2BLSG^@YUsq2AnI%Z8p@IT7?5$|3ZrQY;&zW${U-;I z4|6+dXF3^>{<-C;ixlG9c_GPfYiaN*NBTjE3gRYd@jWjPrx^Y`CUy2J8MFnjIC!G2 z6g9Lf#L&yKa`%%S5RVw@g(1 zCc^xKRhn9L6i|J$Y|E!;I{1vpIc?tA zXif)koJvUwWkBAW2H~rt=sT=VZ_`2lsOpFE&BMrZ$2H`9&KEJDV4cn7217Q~jejqW zM|`KA)o1E7We+~-&+j@a;XS>TTE7)>z{VG`v(DRykhfueTU`$g)>vv>Ejq)3TJ=%E z<$q9*J=~*8IztE5t-IdJzh%MD=j>8v4GO$4kxqSFhCDKakNf%s2S&2xN1k1vz|6Ok zx9yS{uynu9#ZlBPACH}t{iIC>s~)XJ&IJaz_xFgVRMX+6Yxu!SVGb~|Gi9+r&jJ34 zc59x<;llZ$ZNF;wkilEty>omy4ZIg=9gS8rxZ#*_V-occ&nzqLlow3I%^#NK0vi|? zZQPC?p@7|8ztl?XkH=!C%5Gm_LkB@XbP4j2D<8)r3@@15FUk_jd5B=3 ze0yv?&c&}=UPKY7Y%rgEtla&Q1D4&5Cb-`iqGXmeTb+{29F4GhsmTG)$d4yw@aOSAAJ+;zON7s+q6*TfZ0HZm@twhXZq8cHsOoahWZ-iaIB7KTWMUM}+9a&4K?Cu&!^AX9$9*V?WMc?-)pdv0b^>ODoX7 zNUM7gI*89bY}_K(L4v_OrRYquV06o#4!^xDkd08$5o)7?{Iyw1=PvA1yoHkHIB&g+ zy5lDUNnpyBu?a>z!k-z=MY&1x^(aJa!xFoI-qVru?vZ(;g-ioE#ewvmn6Mx10YFjT+gUU_nP)Ce(JM1TiT(&5U#zR@V=M@n$-G(hdwi*Y-7;#hK+Wxu=T1ED$MZF z-$eaUI2DY_w3nFfumwL&{tKGezammMbbLpgmD^T2V4`6UT5p*D3=!Wf?o9gqu9F6` zKU@4>yr)3oR}E681`Vz)_w#woro*Mb5`1mr$RDjXuHCnR0p;I4S^qX7&N?3SbSLV` zH#Yo92;5Bvk7v(xtzXiC*$_Dr>q3FX$Nqnws3BjAbgbR~70>w;!*50y0S8wnU|RHWzv1M6sD)E*%6-yo%hk=#bK+63}{q1q%=S_o~D@ zz?k>fB9|ugLH395SZ>V#kA=Ez!FQ~g+n2mYb%7BFor|c7M zRJh}QxOXS^%`KUgMA|45K5i{Keg6{$ZY52$)#1nnW%g`Y7%%3_BFBbtG8rOTYJiwgWN>Wj^D zSfF^0-||2Gv9~o}>U}E}tW_TP?mL3~`T}P$SdtBG$=8Ke)6tiSY#@)8(t%H3Mq@SV zA?Kscd8%I^!{q|+MLk8tUxMRDKXo$U&l67`i_fSJWL+K3M7|fn@vKirJu7RVwD#$1 z#6uZl_Y_b!SZ_7@X}dTFhHjE-E};K+ne$oyStJ(>e!k?Js&#;s*Nq!pqS24w5hKj3 z!9MUS|50)V8yZ5AtZ$o>q5gx$r9BylcT4?V|3-b~t5^H@J?!rzoAiWc6xhJ0?E0oY zi35R6fA0K395v#`H7nM|dDFkHQ5$)y?P0fpJk+}^ri;DhfdP$;ZE|ccCK&Gfr(lRU zGlN&)M0P730xTBZ{->j?52z|BPp5#3hNu2E#FY=#L=Qej|6z+~Hdo*z1uSEV?hPM6 ze}FW6t;d%RhNNL{6Jsi5jyAQNN8D^J(I@j2efFe{*IG%51aP&9EYVy|1+x{DHM6l; zN7>}!nOh|AjW&BIwx14ux!a_=pVGl>*P-c09|=%bv?`^m5QBuI^i=aG+!j+N}oW56^wOF`3}2npHwdcr#B zM~#XZl2QM9>b=zbb14Pltxr9rY-a#Zw!_(j*yn9Ceh2g-J}l?c5h%?tTfI-To(Lvi&SsH~Xz=XU(qGr?kzW=wS87X< zAgG1tJ&gTC@w4i=n~_Z5>)1*-g#WJJyK=>4)ZL;C)U?i_&++}l`|FchWaupk9NJ|_ z2b0y0b5?J}^NBcYN-!lsiQ>9Vn=ntH8r_tZUr&dI&nfq*(+TxN0d{uttuhW%eihsggH>x=guAv->d z0G&TCrT2=^pm+Y+(50gkkR^4F+W?C@ZoDnOq`S1tNi`V$7^l)ohHP^3fE{^Noy6XW3%@XeY@wVFnVnM&yYI{d|bOsY?IjVGjm&(wGjaZugq6) zKVpCV*O{Eop~2brjSotf5ujbzM_cC_8FAmlZmDAQk4^fwkj|kVS+~nd_ALWMQj<;| z3B(-1v%$8Tc#iy2dK)%8rNZa!cVc-6NPFCQSsAY+mo=3=!&Iw#o-z#d+TSf*co%I5o7-CenxjnNhdy zCSvYLWkz$E;w>8RkTiK;Es$XGqq9twG7TL_w2_sJvm$xC#{6(LwY-s=ICK8y9gap!Zm=G1zJ~@p(tPH7OXfT>k0-UN(tv$9<7w$qTTlpI z);xH_4!XX@CQ*9`(EmMqSP+vMZYpzOvZi)$XXU>J3kDfxJeL1E-pU2ufw+JR862qX z39aQvUbbt@`Q2AT2Z&m0rf?*W3sbA6o>N_s*JnF68G1Q@y~^)=_s6IUTog{Ybc_Y` zK3~_V2e$BFs-`Cu@m)l^`CWx`RH(cfw0CL)`dEJ=&Q_3UFq2H%Al=J=*TT+`V^$pa z2Wn&5o48P0>J;ND#D%K2+U{{%P&XAfF1aJhg(HhCDq?rJkiJr|)Wn|x^V?K+T|&Kx ztKrhr<%9Q-)cDsVfDX&9T={WK3iC^^_um_uN1rzB;=`?&$1q8u=1bhA0^4+%%45`V zBY^+JGvq%Xm$4&-P!}*uZ+cy_75$vkyCtSk2l){{^@%4Nd8~zemOkRNf|}`}Z~I7) zuKe82awqnU=RA|G>1;Tx7_MOMNdk-SwYG@@6fkNke|Pd1=5AC9x*|5AuhVp5UHw7S zwOQi=GN}Iry2*H|BCl)ayB(2){ls}o+pGnZ3o`{1FU@sD2#qxVhu@dn*SR zOl=-~P;C!U9orf9sE1WNvAjMMhk1=QGlwz6nbqOu2|0;l<#ikF7cAjt0% z4}X$9Se)%t?|4XtqWv4=@1PG=8EI(!68X`@($a^XJ2;@Y{*L?jQZ^Wg-iX^EfjH7b z@8v2vHt5<#1vsMau`I9HVmbP&4@TGXm?4jD{BeJT(T4o`qDcAC1k9n`&0%jo&xXAl zhZ~b6I8b%>vY3pb13Xq3Gw9_wfOhK})_p-okW~{anz)O84nK>)j>Uc+)!ueHo(24} zPG@TopNKj-srLP%gUa~JWv9Y0A4!#tH@QRuqfncPg)};}bsZh`M1H)*R6|wj9|wL5 z>hIcd0ddbYx6gA!9M}?=bb*+Fdj7S_!E5U=hvEFKDrpVkN0~kDSw9#cM|iX|2lY@F zZuHoy?G7-$E5qg%_Sdy1B|_VeV9vt)rbcTK;!$16>uS0xL z5f#4PUYY?3584;6U>>ATEGe;a3j-L}FUKB7{p*=>wZNn8Y*@Uan#TBoKJ#l`;-C{7 z)U2yRZ(hbcr1;u1p2&9%w)1vLqyI~>HQ%5Ui~OA%_IyB&1UIBYcXqx(e4Tnpc@Fs+ zDch4_`-==58jq)KBpnib&YZ8TWP^Lz$+%Fw*CV_#ckHa$;MkkFW%e}chfTzt+F?w% zAscQp-Oqyi_Z6Nis$t(*mUq_3j|t39p&yTLVg5=?;`(0fSAHj(4l7_jK%K{!f2AS~ z+?S}jC7aNou$kyTqD}xksrMxxHj^Q@dx%dIb@;ZL&&*fow-=Y}l6PB6hv*>FF!ld( zeOBt?;jsj8aUSfg#`E_qxx4Pm2AoS0&zhnW%t6e8w}WS;E#$;|GMmLav7Jqwb;O-n{WC`jz|(i~4z(%d}N@-LoD2 zJiLE#b#tkLvIalFES$BX-9{$LJ=UA)pL{5c28RV2m@O1Z$7u;-7j zFBzt{+(;b5b8h)c(KvAy^SyJ8=hN`MrH+~!j-g-Sbxxo|X@&|eoagCTKM@y1%tRTZ zzjSK6Q$jwK3Vk*C<1;I1kfT(i@Lqrc{7T9#N0Fay78z*&y8`{3ME)$Q4hIgGowUkB z9pja+g{7@B13WkH5tKuIM0gsZNV>{^YXX}(2Ua>lDqFJq%u6m@?_=F*UWcDM_`{P? z&g#`MOz+#ymMb(0Oh@8sC3 zAs;hP!w4(t59T!Ai6&89U z53>BqR=RPA4&Bk`R)1x2K=M)FWzHf6$|97m$Dps)e^tb*)XV|i?WFweQKdqUUY&&L za^%Tt410H$FhIYme~YgQ=2AiiHQp*9Ke2kVTmJec(7$n;<&DF8 zAV(*@-?o(k5VS?)f`8or<5(yKOJn1?&P{K>UA%zq|sb*5lW zct!3rM_Zi31r}?M5piz0ocQ(qXaWAcg=IdSNxWCOnQuP* zm+Kn~da$Vk=h6bF=QAJXgSXT_`V@|N38r)pzwBGoF(#+qh9h3w{m!~b?J(94?GtY| z=BZ}J9@WQU{)S+`{zG{`2ZUC?C}byaA>i#`VM;jW5Dvs}V#d)QTtmFNN|z1a%WVUs zEdKK~?_B*j$Al|)Hu5{RVD8@J-}Br0Bv3kf&b#|E71r+J8y32aenF+H!RHbpI5!6E z%Pd9RWRF+aZtS!Dx8ofIP#;ivF5bD%iUrgn86gJt;h2=VlcL{nzaXJ^!yfNcyAWyr`KM<{Q9T3iZ8avfeT9euc^xSp#HnC`*SV&%?V7U+C2|AVAwrD zcfXJPlHWSr+m8j_JT*-bmpKs6ThTlmLLg$tklNtmr6AU^9!*BjD8{kEQIXTwBay)tcbcsm`!`sdHa zq2H?LG5+4@1lFO$(1lN{FeiH7yZIN^Z^^RT?J|fL9AYfhej*-nexvTx*203js{|=S z&j{d@YZW@DM1mC{I~ekm0`h+TOS~f~Q1EJ?#``E8+Px^mT&%nF7uo*XFfY`yp}3Gc zNrPhghaS`03=peLTfZTX00R2VGeSc%>SSZL?{{BV+%I&u|94(koyWikrAq#U0Ito30yD_I)oLZ-= zPKDiTWCqf9aWIV;3kG^k+cwShn9Z`swM_T^q0TaAEXDxJ=Ig2kfJiTqJRhIni3G z-0}TCJzX)FZ-f5ivX^|(i0juyKb&k%qCrZ&|BM{!1-t_Sic<}ki(8kuA)(m;g0vEB z&WLm2a@^GVg^ideOa7KONFzX)&?$ery<~Vg;3>SM8~2AU&uy7ZLY$FVd}3S#`+z%- z-)iJLRuLbbo7aT(3V_JE#y8wK|KKPJ@mO0!EWe^vlHGHW&mV-d<%r zqp*btqJ?2*V^2vS=;1bV{Tvy1;*OTvA}`61{`>Y6;?hm-%G7V(T#(xuVm!{K!W<{d z{7F0NuG{%%e-?9KsZVf$$T~Xgul-p28s~zfc3H<1=B@7hUATAxv=FS_5tfJ z6PNPkm~iAx&FvGIpLqSiJ5Lt}$7vsrxAJ`Q1GnfBrpkaQk z!a#K|^1w|3H&4b-Qoz8epsVsa6LH`9w3c@a`2EP#b=Hpxfopa5-)^J>C0M}E41E;S z_AO!2C2R=$SUmLtbHiqt|Fomb(Qiu#@H8`{Lvr%b+o|0w2#dDUTR@yPBl7!)w=NTY zI83F{W7u$dpnc{tg9|~{Pt?A-5<$gb_5$@a87gmHe&ta^gFu(IX9nwOkic`9SA8=R zz8p~_{J=gdxn$K3DcnO@Gdb_#hJKXhA@kCs$Q$?E9>|z|VF$J|rR5Ko<9lSN#Uxwxg(XZ`(wVy@vSjf~zR2&jCns4lL=T zRIpc;KctQG*RM9!qeB__Vo6WN(*x)$r|%U?R_DOw13oIxFn5$0m$s*O85Q37{bA^C zW`Ife0jMlwf%su>VkGwKogv#K*F8ZWYSpKUwl)r6sIa6r;2909);!uLtx_Yrsx8?|bky3l_eJml|~hI36T^_XD;&NbT&9to#W z_i5$-$+DEj99EKwmsJGrPt7Ua4#iw@ug%giS3WYlDEb#EkNj1%PB~cUwmrDz8Tp8P zB*FJLIS01#&_U!kiLc}f?rV6-%Kn*Vfr&f+sFw`~N`{4nmqlRyB=7j)H5VC}Ydp3&iyszzn9LzV=aKi^EY5`|fpM zY={C2$-0qmiRioD3KuBjr2xsHrNb3-%w6 z%-60YcidUkg*n)B3AW$s(AO3_cIa0D=3K@4h=JBD=sL1d*&q9~&dZ-wbtR}51wVgO zJKzAsZwJrpSmFpw<*X+xD@Uk(N=M-9W#f$>?!3*0%J8}?uL6;;ScYlSKA|2(uKm|H%Y=Ui zgF{NxaKElWSek%2Lfvio8&*eP9@t+zt>-%#uG8x--%X-It0B=dxEFE4$nN?3^5|E@ zUOQwY?EtUG=Rz{V9bm<%e{iuF6#^m^cFj3szNBAy?;-rTzRTYm8Ra-YxK-0R%TPz$ zJ6SdURO>M0SJMe^l89j2Fekmk75&&(v{pe))C1f#s)y%oK!^AEzrnpEV3gey8Vy4| zgzLl27r;DP-lg^$8?9a@n+)|8*dy@;WJn9?ES8P)PaWHBTjW4=VaAfRY18|LpTbF%+(fNwtVsQo1pMCEw0>CZ8@ z-)m-i0`;nx@P(^f)HCc=niC7U(ckQ4ZxvC%`(-!wI(6I@yfZcnyDA|rTP!$I`4PQ$iApOKlzw>KoeIZ%@~E;z<}3^A zJ;g6Y0QD}HhzG-(R49{jc*sV)rk&MpAB%YQX`kFxD`nh&4RQLlxdi)aPJTra`qrg) z?N7I-p-yWsIgxn40n*3a_XYK19{a|xB1O#6xGFty)kWPx!HXD0Z{@(1{5NAJS8*RV zIp_8CQwLCcwX{tId1qX5X3`ol3Q`!uAcx} zF-O-q>_+{eaM0ejf({bOGIQTmZK3sOM||G{dr0p~)$bgjz*@JhsXgss4GPrk&h`dGdwVyqPyB5D`)3JcRyA z&m*<{7Dbrn;jk+*c&V^4O*qdN>*V|p{Jo9!U7XE`bBv+)?F`W8`yyB&U;O}c-H(=EYQsI<{MnYqGKw8Xg^^JYP@iwz^;%@xx=$m4&XDI36ik_+0E??)eAzU)vvUJ3fdy z;*T zUredafDFf0pXdGdodR!)*Crf|!#r-Zkp39z@~O)Y9lM1%`r3(c!;5Rs|Ep=~s!C=- z-r&ftg0CEK{NZ+PY!mu4!NLy`rViQb;OSg;XmUgj%hkTsC?t7QZDLZvmMS z@xR>RFZVI$`128+?)jhpw99X9T#61^Z?zWXQ70^)a{srB!GJG^lVy!JQ9-q}X@fHI zz4Q&cg1ZRF7uOs=aqJx#tQ}WOoZW{x71&09j(Fl$%i`&@Mf4?-Z<}bLE^iT-K#alj zcUIOPY()LJxmD~zDCTwC|87|svV#esC9Q8R4v}GsIJa^b?~nFpQIC(fr>nPaNFXhr zh&lYpBzeTcPeEZzYeb|cZ5Ipon&trq0XkeUZT?I zFo>O$Nh3}khT>+G#+T8UPdXhdN5q_!*!JM?--x^9_j*OB>vJJipxQb_nggfF6<*mX znCm<Qvs)(rPkwU)d#kHI|1#f(4J_swWfOAnqH#e9OM!({7F59a#J4CXeV z9~7WieCkOY=8X1l{95ceKgp+@;pjsC#6>l$s`o0~I8XDFbqp5rB@Sd03|8!G={9?Wgbo$P3R zXWi1~BR-bM-3hnsxN`u+-W;Q2cd+nTqi-`6`_+={+rq1E;7g5#JwKl@~2 zdjJQHwQBavBhOX0sy_1xa{-f8Zijc-;OBXAuG0YP{%!*G$~l}nQvJSVR=CG;-N4W* zY=8mxEj9;mP!HL;X3fzRh%?(aYG085#sW*V9fJd>GGIONNd6ql;eRo}c<_h5mc_m_HmRZqAZn0H6Bn z{xVh6bN|RH6uzQLk-Fzr5!M>Qqj;HLe<&Y*^J4M4VW|fr_ige5&trq4)WK?Fr1G=2wT!#>%3; zruxXp;Q<%O-d5FW94D9-oxdI;b_C4E2;cv-AAuy^jr3NmR|auKheQVE=&pIRMrIFDW0wn@Cc&jCB`sOvd>ItV;-&<^UxJdEZvEeZRLeOO;)67rSZ zMJFZ^r?M>_z`Oa={r+mx$w9c=>wjsI3Lz`|7&^0fuCEi%&il1 z1kxza&$8c4(DscKtVUgOI`VaeCEhEawHA*Ikgw0Em5E89-u307l!nME%$3{P?cRX= zz2#%;s-hGX>^uknP#Org2BOKO>?e`!PV0&VLRc!+iBwr0b zZo<6_{ZP#TBO3aAQ{`rFS98H=+pAr>Q1=lk^C)B@ei*#El%Lv(cqmPk_e}`yq4VzA znP-FZV`1O=*~h3m#jY^=iu-i@^Nd#MC=wLEPO1z>zT)&1oRw}N&poiN+!1rfIiW#u zPrT7*zy9v-uqXOTRx35vU|qdBq(uo+Vt}94$o|?gCajeFA>~iT&$VxjIr$3Sd&Mg8 zanxHL-n;+V;05-_c-1S}|M7O7wcJ5nBuJO9S_gHI#NfdPLz#$MrXr84 zWgs6hi8YM|tmu z?kTH4?3umjFNvR9b!!~+y4f2ZQO_tMkP}W|p5V=8+xAe* zd(2hUm^p>oLGtdB4W3xX8Ft_9A9;*@J8NCv58OAKD)*{;xtas-gM+1SnRDTfmqUR) z{+wffULIr!qb}(%Cc-y{zMz4R;hz6ha2<|R_H8&SMJXf{(m)C+31!{M4sq;b9OpZLM56e5}FH7d!BLPg;X5kf;grDzBVkuusTeOG?~9?!Uc_cd>U_wP5F%Et9=E|cyZ z1wUb?;d4`Oib1W?ISW3(2c#I2{N3P*1Bw31;Y)B}QpNlkKDA!_|I!)4y2!EW?bQ@E zyMUgA1+z1Xy&R!yd-QtXx!e-}w)N0BauPqduY^hf-x_yMC|9xRj$ezL9QMf%g5IL$ zztA7yxg~55^i~g!y6vtBSGv+P6FCI^LCIxPd;EXsAGr7YYfm_fZ1qKaSRCwY@$LG5 z@y#br=KyaO`04fOA4h;oq=Rqw-2;yp({s*HGz@-@YsoG%$e}IPH$K9%5Bq!5!0uJZ zKle2TB~1K1e~$9$pYVgP;iT?yqbLR)R8`Ux-EU2I=GXe>YuHi8+l(qNe zIFVt8sDXkHbcNi8$|`(^ogC(m4n6QZ-j{DH0Vm8ps=Q!HfCKGcTi%`7=tM>SD%+f} z|1dcPr?s5i$WvAQU@N|tSfiR9vy|BMk>}OcrG?0qgbV3Z?E$}{y(#w8Cwq#k8Nam$ z{OPu^g+rgf?+Fz(1W0Xhq?>!>7awkQBSqDcSw^XDlwZ3oe_Gm|Jm0ieY<+1@HAhxk z@f0BcV)-<90YCL6P6ZW*u_oj5n49?{PQcbpz@B_@XAF<;@t}|}mvd8d^+VdNi{)RtpC_GlC z62EWFoIX}BKa-Ysh=o7Yb*4|&2N(ZgGfK5be)!*m8{^)C$G!Sg zuKybN!)O7C5 zEfw&^Q{Vn=#(S45p(q-s=R&8I#e^G>gIu2bHI!F?MVq)EoL~10{6K}RQfma0&PVPu zY@XvxQkT26xo5NJWX+vB`@!3?@0Ubt;5_-t=daZZ9bWu#-s6u+=JaqIPuEvrODey$ zF2Mr)hyjBHpB_Q~MWMsq4CuZdqp>Y!%dE+|=%MrZ>l|thljh@XWK*Ti8Wn31HaXj$ z%UJUYxLHN`Tu~+POm~LZM3pOjFXfm>VlNnaq9APg!HN2Z|8?&!Mej@5o_o^J5$7xC z9F%W$B}3^AYVu_FE>x4U!up#4!BYf8Uf$jzQCFqM<8I^7Y{z5c& z=){;N@@ErkB{nkbNR;ixTcZuULi~Pu0mwC&N12(L!@qigKS@jkzMUe;JAR^Ua<$4I ze+$2K@9Z^)V*en|s+HuMnG8Ly$zs;6eYWHlDBxxK$DaDFW@j6TW4;w`4=-b&|Hr%~ z&l-F~-R6SvwQBG;`!8sSvIVcxn;g|Pg#EpDO|V`u{5)+6tFrL@UpH=gF&Dhp2~WA` zMho~do_w97uWm!qDqo8lYV4`LE+~7Y9Ol}BEmd@y4S(LR>Wg+9;EA9n?qoMQmeL%N zY7Ks{Q`w^fb2&!AY*kqy_Gf0KQ$xQc`IU@bsOJVhsnTb8Ki`2CZIx`*XfUPpveXte zUkmCvLGLDXt*K_RdU=^5{FFs*mqb^(Q1F}Q&ib3--zeo>I%g33(1uX{bmZ=TrMs)& zbakS;A3rzj9I~bE>TedOaK6c`+m72S4m~bErFT5o4&KAMcaLzrBh3VrgzbmUGpi{6 zpG7_J9VC}358(U?j~*%wtw-;Xp_(Eu__{i;&FlOYzz=@N`KZtX>|N1Y5@J(aNNx2s zlZ)l(!C3kH@Ip%_DSWpUiXLZ@NV%H*=??hItPbnvpy%Q71GR%v@Xh2+v`c(1!yYGW z_<5OeIoqivZr&A6R^+RAqT!9_=R3Z*gN<^{r!hSjIsY1 z4CHbTqu(Jb!_gW1dQIB>{P8uIHyL54MrP1=W|<!d( zJhZutMV)#31mA(DKH0ld^*D6HCvAcJH-IPCFXOg2X@#8TFiqHCt_m*^FABYf{ZlYS zZs>+RW!GIk)mjUj?Pe;_2%pr+!((Eq0XDRoYg{%9__30+*4B1)kHhg}se7DB&*GS{)az@z-LCKR&p99e2UK8$kcNJrY<&&Pdes4K-gWik~$L78g@aVN{^Nhs?G+l1? zJHf}0B4ww!h2Xz8{~y!5#uWbZ<=NaP;vCU$a$nQ06FEqikSTlMqCM+Y`#(-XP9R3M z!2o`oT?U_hpQHaxvbD!g9eL`u6*EKSZ=Fc5CF#fZ>-MB#ny@373;NPLi8#CM@DH%u zq+{SS)?|!+J~D|u2)7Qejx)&Pe2y3lIbchCD~ILuMxg&UZsXmBzhBALP>ytPpoKyk z--n@BMMQDT%m?qu2j$beodag1qi_GlBixc+txuilgFYA~6PO<44xM4D_)k^~`b<<3 zpZ$mT_7%U9Rk=hg$fu!QzV(dHv`^)dF}CL9Jky&3Il?(YKEZpqC}jTT%-CZXVT3?YWBRTT?Sg zkXOQaX;aS=!tF|~jGxz^mq54CPRw`2e6l;`deQZU3;j_Xw0=tPv&9bD@H|3pNM`G4 z4cwP#ot48OI3H`4)<}-QXQdN+ZF~T@Iq}+suI>f!LvIkw2pa*P=)5X8{=Ey0{(k4j zfY0OmwcqJy;E(0Cb9wjk4DeRwRN**uEwv`qnf$ZR>%C4YJR4$Bj^2&v?2im;`{&z+ zDNSptd^4-CWjFfnN~88}M}DQRH-udde*5j?gEOz-EBV%bA~c{7I6=Vh^$yR!zA62~ z#n=zx+qqogDy>NBNU8s$CAM@-t1wIh{Wp=`OIBTl&nA9IcUheglcuc0(k(8aM@r+D z+B113Z3vHFdM(9?{8~26)opU3)2h>J)WoewV>)E}f2uY#KO=wfV=L%JA%70ca<`}O zkgF00)3FaS-DLd0M=Jc2NCGX;LFG~fTK@%3@05!TM!vG3aDPOuzZI#O`|fajY)iiU zY4en^KUP1h`TRB){;wMb+{ufOqsx=>NPzD|r6?sg_NP7ZRh##9K!2%ynRdTD54v10 zmu~oa8w%Xis$-~+eyFbpzBm@6-=?wN=i(pu#CC?N>Z(ESZTG&+4IU>h`Tl~t#SDsd zNv-M^vW1^GZ^a0DL=vVZ)q{|q*BTB9m8pP#`c~B5v*1PB-M8(k;I*gV2VX3$!7s1V zAM8o~AM`HmSpSDl;VUrw&}WSEH0jQs`4#FMN-GsLs`ke|5mUnAUg1Km(dV-~cd&^i zc|q3n9{ff9@86%p`@>m~<6(FU`y|)?bK<~xUpt2b^nEy_pmCwJ&5BKJc{WmM@td(ZwbW*Kz%XbQXCk z4vL2YhbG?-<{jGaM(b<+NC|szPt)cI0R!Ynmp$4k%nw|mx<=7rhXYA>TE|_>2JQ;w zRaOmQ(JL3@^-qHxsn7gz!&A)jgFYo4N}6Q@EHB^%u}I6z)GP-1`(CXc%C{wk!1h}IdgvjN7Q(SlY$!P- z|Bep6%W7YTxf0-?&S)7ZynPEDyF__o0rvGR*Sd3$XCQ}r@XG@+_}^qqRCl~EaUnOu z^Q|S|jm&QZj9UI=(KGf^?M6`+tsCR{_6@$mUvvEgv~)RS;3ySy82rAv+Rbnox^^r9V|sk*=$LyJ~<{Nqj{Yh_$_{eixC z)tO5Y^TBWCXps*1#pFrtqP_~?z!1GxQYMgYC3JShQsd4reHM-C%4tn9?CF5!h{B3f=p8t^x4_~s^p=5h zN|#F=sO`%0vtiJ0GP<`u?8UrYm8v^8893Aa_1*r67~~A}7I3p#z&9#h?DyMfOOY zpRdk+KHwLsxl*(RIV7RF;*?rE^i=&MhZ=eKl#ic_^#)FqRaxThzuT0~kEl|pJLhks}PCEza?^?0iy3!VupvRfe!hX^EO}dq3%Aq}8 zUejMM+aSNT(Q03?GZmU|OO61(l*^jGpmMnlWlg^G+I;`7Zhlu#9(lAVXKz2Zbqw-p z5s59l;7rqfZkssB4=$D~vDa_5hYmCBIfH#LyWq}$cWBM&3z z%${h0Prp4yEX@Kro!bIZi-Mdu^yH0v@V12<>TxsXE3CC4Ef0s&JZ#K^gn_Z#F%Hey zmG7GzWK03?5Ao0XyWhs@OvXLtP)Bu7Vs#txHiFMDRa?1`s{4&eOU-}!x=td9-w;<4*WqE9e?4o*a_YQX;g=bk3l2jI(qy^B15;{1K> zy;|sp1)Wt`B;TKiez?5`?rBljOJ{~7T>RmeU0B~I34fJ|d@1i$+_%2pO8+I9!RNoV z#=V6fe2ofoy=)bS?pF!C?0Jn`jI4yAZ7B5fqmkv;CD1pMaD{&h_WTX5ZGze0Pa+x> zL|=kGf0>pdvFegF-FT24TYa5LPtPtrpuPdTy^^2ru0ktH-R%>i4gVQ6i+8Np#il@S z)h%nG3vV;(t$%5RePrHc^|_1SKeQR*>%n_2X_sUY3SPD=C+;~{uqzcl^?Ff1Yy;nR zt?OxC2YR>Af`0<@NqMN{!=>%$n``^AlQY63rjw`Mm2vFJ8a2%;HlSC6=XB1^<;eNe zhnF|Cm-9Z@$u+2(|Yu*!3g?3|9rLlPJ0%8uqwJ@@&Y|?9KJ8(`tV^j zKh_E3LT|-)R#?kc2r3A;Zi8FQg$=<(t@_N;fFWT@K$<_&vVa{p#e++k6$f8wH7Wf6dXHk=5TYWyDiiN1@Iw=V92e@=YpY}DTzu$7nP#ND6PWTo71qBswJ z$Fx(1MXifc#(|RsI54Uf7{aVV-h8SN)caDB? zpg`@qwF} 4: + raise ValueError(f'Input shape should have 3 or 4 dimensions, but provided {self._shape}') + if self._shape[self._layout.get_index_by_name('C')] != 3: + raise ValueError('SyntheticImageLoader can generate images with only channels == 3') + + def _download_colorization_model(self): + proto_name = 'colorization_deploy_v2.prototxt' + model_name = 'colorization_release_v2.caffemodel' + npy_name = 'pts_in_hull.npy' + + if not os.path.exists(proto_name): + url = 'https://raw.githubusercontent.com/richzhang/colorization/caffe/colorization/models/' + proto = requests.get(url + proto_name) + open(proto_name, 'wb').write(proto.content) + if not os.path.exists(model_name): + url = 'http://eecs.berkeley.edu/~rich.zhang/projects/2016_colorization/files/demo_v2/' + model = requests.get(url + model_name) + open(model_name, 'wb').write(model.content) + if not os.path.exists(npy_name): + url = 'https://github.com/richzhang/colorization/raw/caffe/colorization/resources/' + pts_in_hull = requests.get(url + npy_name) + open(npy_name, 'wb').write(pts_in_hull.content) + + def _initialize_params(self, height, width): + default_img_size = 362 * 362 + points_coeff = max(1, int(np.round(height * width / default_img_size))) + self._num_of_points = 100000 * points_coeff + + if self.subset_size < len(self._weights): + self._instances = 1 + self._categories = 1 + self._weights = self._weights[:self.subset_size, :] + else: + self._instances = np.ceil(0.25 * self.subset_size / self._weights.shape[0]).astype(int) + self._categories = np.ceil(self.subset_size / (self._instances * self._weights.shape[0])).astype(int) + + def generate_dataset(self): + height = self._shape[self._layout.get_index_by_name('H')] + width = self._shape[self._layout.get_index_by_name('W')] + self._initialize_params(height, width) + + # to avoid multiprocessing error: can't pickle openvino.pyopenvino.Layout objects + self._layout = str(self._layout) + + with Pool(processes=self._cpu_count) as pool: + params = pool.map(self._generate_category, [1e-5] * self._categories) + + instances_weights = np.repeat(self._weights, self._instances, axis=0) + weight_per_img = np.tile(instances_weights, (self._categories, 1)) + repeated_params = np.repeat(params, self._weights.shape[0] * self._instances, axis=0) + repeated_params = repeated_params[:self.subset_size] + weight_per_img = weight_per_img[:self.subset_size] + assert weight_per_img.shape[0] == len(repeated_params) == self.subset_size + + splits = min(self._cpu_count, self.subset_size) + params_per_proc = np.array_split(repeated_params, splits) + weights_per_proc = np.array_split(weight_per_img, splits) + + generation_params = [] + offset = 0 + for param, w in zip(params_per_proc, weights_per_proc): + indices = list(range(offset, offset + len(param))) + offset += len(param) + generation_params.append((param, w, height, width, indices)) + + with Pool(processes=self._cpu_count) as pool: + pool.starmap(self._generate_image_batch, generation_params) + + self._layout = Layout(self._layout) + + def _generate_image_batch(self, params, weights, height, width, indices): + pts_in_hull = np.load('pts_in_hull.npy').transpose().reshape(2, 313, 1, 1).astype(np.float32) + net = cv.dnn.readNetFromCaffe('colorization_deploy_v2.prototxt', 'colorization_release_v2.caffemodel') + net.getLayer(net.getLayerId('class8_ab')).blobs = [pts_in_hull] + net.getLayer(net.getLayerId('conv8_313_rh')).blobs = [np.full([1, 313], 2.606, np.float32)] + + for i, param, weight in zip(indices, params, weights): + image = self._generator(param, 'gray', self._iterations, height, width, weight) + color_image = self._colorize(image, net) + aug_image = self._augment(color_image) + cv.imwrite(os.path.join(self.data_source, "{:06d}.png".format(i)), aug_image) + + @staticmethod + def _generator(params, draw_type, iterations, height=512, width=512, weight=None): + generators = IFSFunction(prev_x=0.0, prev_y=0.0) + for param in params: + generators.set_param(param[:6], param[6], weight) + generators.calculate(iterations) + img = generators.draw(draw_type, height, width) + return img + + def _generate_category(self, eps, height=512, width=512): + pixels = -1 + while pixels < self._threshold: + param_size = np.random.randint(2, 8) + params = np.zeros((param_size, 7), dtype=np.float32) + + sum_proba = eps + for i in range(param_size): + a, b, c, d, e, f = np.random.uniform(-1.0, 1.0, 6) + prob = abs(a * d - b * c) + sum_proba += prob + params[i] = a, b, c, d, e, f, prob + params[:, 6] /= sum_proba + + fracral_img = self._generator(params, 'point', self._num_of_points, height, width) + pixels = np.count_nonzero(fracral_img) / (height * width) + return params + + @staticmethod + def _rgb2lab(frame): + y_coeffs = np.array([0.212671, 0.715160, 0.072169], dtype=np.float32) + frame = np.where(frame > 0.04045, np.power((frame + 0.055) / 1.055, 2.4), frame / 12.92) + y = frame @ y_coeffs.T + L = np.where(y > 0.008856, 116 * np.cbrt(y) - 16, 903.3 * y) + return L + + def _colorize(self, frame, net): + H_orig, W_orig = frame.shape[:2] # original image size + if len(frame.shape) == 2 or frame.shape[-1] == 1: + frame = np.tile(frame.reshape(H_orig, W_orig, 1), (1, 1, 3)) + + frame = frame.astype(np.float32) / 255 + img_l = self._rgb2lab(frame) # get L from Lab image + img_rs = cv.resize(img_l, (224, 224)) # resize image to network input size + img_l_rs = img_rs - 50 # subtract 50 for mean-centering + + net.setInput(cv.dnn.blobFromImage(img_l_rs)) + ab_dec = net.forward()[0, :, :, :].transpose((1, 2, 0)) + + ab_dec_us = cv.resize(ab_dec, (W_orig, H_orig)) + img_lab_out = np.concatenate((img_l[..., np.newaxis], ab_dec_us), axis=2) # concatenate with original image L + img_bgr_out = np.clip(cv.cvtColor(img_lab_out, cv.COLOR_Lab2BGR), 0, 1) + frame_normed = 255 * (img_bgr_out - img_bgr_out.min()) / (img_bgr_out.max() - img_bgr_out.min()) + frame_normed = np.array(frame_normed, dtype=np.uint8) + return cv.resize(frame_normed, (H_orig, W_orig)) + + def _augment(self, image): + if np.random.random(1) >= 0.5: + image = cv.flip(image, 1) + + if np.random.random(1) >= 0.5: + image = cv.flip(image, 0) + + height, width = image.shape[:2] + angle = np.random.uniform(-30, 30) + rotate_matrix = cv.getRotationMatrix2D(center=(width / 2, height / 2), angle=angle, scale=1) + image = cv.warpAffine(src=image, M=rotate_matrix, dsize=(width, height)) + + image = self._fill_background(image) + + k_size = np.random.choice(list(range(1, 16, 2))) + image = cv.GaussianBlur(image, (k_size, k_size), 0) + return image + + @staticmethod + def _fill_background(image): + synthetic_background = Path(__file__).parent / 'synthetic_background.npy' + imagenet_means = np.load(synthetic_background) + class_id = np.random.randint(0, imagenet_means.shape[0]) + rows, cols = np.where(~np.any(image, axis=-1)) # background color = [0, 0, 0] + image[rows, cols] = imagenet_means[class_id] + return image diff --git a/tools/pot/openvino/tools/pot/engines/creator.py b/tools/pot/openvino/tools/pot/engines/creator.py index 563e983c2dd..895da98e7aa 100644 --- a/tools/pot/openvino/tools/pot/engines/creator.py +++ b/tools/pot/openvino/tools/pot/engines/creator.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from openvino.tools.pot.engines.ac_engine import ACEngine +from openvino.tools.pot.engines.data_free_engine import DataFreeEngine from openvino.tools.pot.engines.simplified_engine import SimplifiedEngine @@ -16,4 +17,6 @@ def create_engine(config, **kwargs): return ACEngine(config) if config.type == 'simplified': return SimplifiedEngine(config, **kwargs) + if config.type == 'data_free': + return DataFreeEngine(config, **kwargs) raise RuntimeError('Unsupported engine type') diff --git a/tools/pot/openvino/tools/pot/engines/data_free_engine.py b/tools/pot/openvino/tools/pot/engines/data_free_engine.py new file mode 100644 index 00000000000..35789b36cb8 --- /dev/null +++ b/tools/pot/openvino/tools/pot/engines/data_free_engine.py @@ -0,0 +1,19 @@ +# Copyright (C) 2021-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from openvino.tools.pot.data_loaders.synthetic_image_loader import SyntheticImageLoader +from openvino.tools.pot.engines.simplified_engine import SimplifiedEngine + +class DataFreeEngine(SimplifiedEngine): + def __init__(self, config, data_loader=None, metric=None): + super().__init__(config) + if not data_loader: + self._data_loader = self.get_data_loader(config) + else: + self._data_loader = data_loader + + def get_data_loader(self, config): + if config.data_type == 'image': + return SyntheticImageLoader(config) + + raise NotImplementedError("Currently data-free optimization is available for Computer Vision models only") diff --git a/tools/pot/tests/test_data_generation.py b/tools/pot/tests/test_data_generation.py new file mode 100644 index 00000000000..75c7a19ba9f --- /dev/null +++ b/tools/pot/tests/test_data_generation.py @@ -0,0 +1,51 @@ +# Copyright (C) 2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +from addict import Dict + +import pytest + +from openvino.tools.pot.data_loaders.creator import create_data_loader +from openvino.tools.pot.graph import load_model +from openvino.tools.pot.graph.model_utils import get_nodes_by_type + + +TEST_MODELS = [ + ('mobilenet-v2-pytorch', 'pytorch', None, None), + ('mobilenet-v2-pytorch', 'pytorch', None, (3, 640, 720)), + ('mobilenet-v2-pytorch', 'pytorch', 'HWC', (224, 224, 3)), + ('mobilenet-v2-pytorch', 'pytorch', 'NHWC', (1, 224, 224, 3)), + ('mobilenet-v2-pytorch', 'pytorch', 'CHW', (3, 224, 224)), + ('mobilenet-v2-pytorch', 'pytorch', 'NCHW', (1, 3, 224, 224)), +] + +@pytest.mark.parametrize( + 'model_name, model_framework, layout, input_shape', TEST_MODELS, + ids=['{}_{}_{}_{}'.format(m[0], m[1], m[2], m[3]) for m in TEST_MODELS]) +def test_generate_image(tmp_path, models, model_name, model_framework, layout, input_shape): + path_image_data = os.path.join(tmp_path, 'pot_dataset') + stat_subset_size = 5 + engine_config = Dict({'device': 'CPU', + 'type': 'data_free', + 'data_source': path_image_data, + 'subset_size': stat_subset_size, + 'layout': layout, + 'shape': input_shape, + 'generate_data': 'True'}) + model = models.get(model_name, model_framework, tmp_path) + model = load_model(model.model_params) + data_loader = create_data_loader(engine_config, model) + + num_images_from_data_loader = len(list(data_loader)) + num_images_in_dir = len(os.listdir(path_image_data)) + assert num_images_from_data_loader == num_images_in_dir == stat_subset_size + + image = data_loader[0] + if input_shape is None: + in_node = get_nodes_by_type(model, ['Parameter'], recursively=False)[0] + input_shape = tuple(in_node.shape[1:]) + elif len(input_shape) == 4: + input_shape = input_shape[1:] + + assert image.shape == input_shape diff --git a/tools/pot/tests/test_sanity.py b/tools/pot/tests/test_sanity.py index 459153ba4d0..92f7d9be042 100644 --- a/tools/pot/tests/test_sanity.py +++ b/tools/pot/tests/test_sanity.py @@ -221,6 +221,24 @@ def test_simplified_mode(tmp_path, models): assert metrics == pytest.approx(expected_accuracy, abs=0.006) +DATAFREE_TEST_MODELS = [ + ('mobilenet-v2-pytorch', 'pytorch', 'DefaultQuantization', 'performance', + {'accuracy@top1': 0.679, 'accuracy@top5': 0.888}) +] + + +def test_datafree_mode(tmp_path, models): + engine_config = Dict({'type': 'data_free', + 'data_source': os.path.join(tmp_path, 'pot_dataset'), + 'generate_data': 'True', + 'subset_size': 30, + 'device': 'CPU'}) + + _, _, _, _, expected_accuracy = DATAFREE_TEST_MODELS[0] + metrics = launch_simplified_mode(tmp_path, models, engine_config) + assert metrics == pytest.approx(expected_accuracy, abs=0.06) + + def test_frame_extractor_tool(): # hack due to strange python imports (same as in sample test) pot_dir = Path(__file__).parent.parent