From 2acb3e1b65e1273deb60b9999de25ecee511454b Mon Sep 17 00:00:00 2001 From: Philipp Oppermann Date: Wed, 26 Jun 2019 15:11:03 +0200 Subject: [PATCH] Write remaining parts of post --- .../posts/10-heap-allocation/index.md | 150 +++++++++++++++++- .../qemu-alloc-showcase.png | Bin 0 -> 9485 bytes 2 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 blog/content/second-edition/posts/10-heap-allocation/qemu-alloc-showcase.png diff --git a/blog/content/second-edition/posts/10-heap-allocation/index.md b/blog/content/second-edition/posts/10-heap-allocation/index.md index f8231114..537a3c90 100644 --- a/blog/content/second-edition/posts/10-heap-allocation/index.md +++ b/blog/content/second-edition/posts/10-heap-allocation/index.md @@ -501,9 +501,155 @@ We now have a mapped heap memory region that is ready to be used. The `Box::new` ## Using an Allocator Crate -TODO +Since implementing an allocator is somewhat complex, we start by using an external allocator crate. We will learn how to implement our own allocator in the next post. + +A simple allocator crate for `no_std` applications is the [`linked_list_allocator`] crate. It's name comes from the fact that it uses a linked list data structure to keep track of deallocated memory regions. See the next post for a more detailed explanation of this approach. + +To use the, we first need to add a dependency on it in our `Cargo.toml`: + +[`linked_list_allocator`]: https://github.com/phil-opp/linked-list-allocator/ + +```toml +# in Cargo.toml + +[dependencies] +linked_list_allocator = "0.6.4" +``` + +Then we can replace our dummy allocator with the allocator provided by the crate: + +```rust +// in src/lib.rs + +use linked_list_allocator::LockedHeap; + +#[global_allocator] +static ALLOCATOR: LockedHeap = LockedHeap::empty(); +``` + +The struct is named `LockedHeap` because it uses a [`spin::Mutex`] for synchronization. This is required because multiple threads could access the `ALLOCATOR` static at the same time. As always when using a `Mutex`, we need to be careful to not accidentally cause a deadlock. This means that we shouldn't perform any allocations in interrupt handlers, since they can run at an arbitrary time and might interrupt an in-progress allocation. + +[`spin::Mutex`]: https://docs.rs/spin/0.5.0/spin/struct.Mutex.html + +Setting the `LockedHeap` as global allocator is not enough. The reason is that we use the [`empty`] constructor function, which creates an allocator without any backing memory. Like our dummy allocator, it always returns an error on `alloc`. To fix this, we need to initialize the allocator after creating the heap: + +[`empty`]: https://docs.rs/linked_list_allocator/0.6.4/linked_list_allocator/struct.LockedHeap.html#method.empty + +```rust +// in src/allocator.rs + +pub fn init_heap( + mapper: &mut impl Mapper, + frame_allocator: &mut impl FrameAllocator, +) -> Result<(), MapToError> { + // […] map all heap pages to physical frames + + // new + unsafe { + super::ALLOCATOR.lock().init(HEAP_START, HEAP_SIZE); + } + + Ok(()) +} +``` + +We use the [`LockedHeap::lock`] method to get an exclusive reference to the wrapped [`Heap`] instance, on which we then call the [`init`] method with the heap bounds arguments. It is important that we initialize the heap _after_ mapping the heap pages, since the [`init`] function already tries to write to the heap memory. + +[`LockedHeap::lock`]: https://docs.rs/linked_list_allocator/0.6.4/linked_list_allocator/struct.LockedHeap.html#method.lock +[`Heap`]: https://docs.rs/linked_list_allocator/0.6.4/linked_list_allocator/struct.Heap.html +[`init`]: https://docs.rs/linked_list_allocator/0.6.4/linked_list_allocator/struct.Heap.html#method.init + +After initializing the heap, we can now use all allocation and collection types of the built-in [`alloc`] crate without error: + +```rust +// in src/main.rs + +use alloc::{boxed::Box, vec::{vec, Vec}, rc::Rc}; + +fn kernel_main(boot_info: &'static BootInfo) -> ! { + // […] initialize interrupts, mapper, frame_allocator, heap + + // allocate a number on the heap + let heap_value = Box::new(41); + println!("heap_value at {:p}", heap_value); + + // create a dynamically sized vector + let mut vec = Vec::new(); + for i in 0..500 { + vec.push(i); + } + println!("vec at {:p}", vec.as_slice()); + + // create a reference counted vector -> will be freed when count reaches 0 + let reference_counted = Rc::new(vec![1, 2, 3]); + let cloned_reference = reference_counted.clone(); + println!("current reference count is {}", Rc::strong_count(&cloned_reference)); + core::mem::drop(reference_counted); + println!("reference count is {} now", Rc::strong_count(&cloned_reference)); + + // […] call `test_main` in test context + println!("It did not crash!"); + blog_os::hlt_loop(); +} +``` + +This code example shows some uses of the [`Box`], [`Vec`], and [`Rc`] types. For the `Box` and `Vec` types we print the underlying heap pointers using the [`{:p}` formatting specifier]. For showcasing `Rc`, we create a reference counted heap value and use the [`Rc::strong_count`] function to print the current reference count, before and after dropping an instance (using [`core::mem::drop`]). + +[`Vec`]: https://doc.rust-lang.org/alloc/vec/ +[`Rc`]: https://doc.rust-lang.org/alloc/rc/ +[`{:p}` formatting specifier]: https://doc.rust-lang.org/core/fmt/trait.Pointer.html +[`Rc::strong_count`]: https://doc.rust-lang.org/alloc/rc/struct.Rc.html#method.strong_count +[`core::mem::drop`]: https://doc.rust-lang.org/core/mem/fn.drop.html + +When we run it, we see the following: + +![QEMU printing ` +heap_value at 0x444444440000 +vec at 0x4444444408000 +current reference count is 2 +reference count is 1 now +](qemu-alloc-showcase.png) + +As expected, we see that the `Box` and `Vec` values live on the heap, as indicated by the pointer starting with `0x_4444_4444`. The reference counted value also behaves as expected, with the reference count being two after the `clone` call, and 1 again after one of the instances was dropped. + +The reason that the vector starts at offset `0x800` is not that the boxed value is `0x800` bytes large, but the [reallocations] that occur when the vector needs to increase its capacity. For example, when the vector's capacity is 32 and we try to add the next element, the vector allocates a new backing array with capacity 64 behind the scenes and copies all elements over. Then it frees the old allocation, which in our case is equivalent to leaking it since our bump allocator doesn't reuse freed memory. + +[reallocations]: https://doc.rust-lang.org/alloc/vec/struct.Vec.html#capacity-and-reallocation + +Of course there are many more allocation and collection types in the `alloc` crate that we can now all use in our kernel, including: + +- the thread-safe reference counted pointer [`Arc`] +- the owned string type [`String`] and the [`format!`] macro +- [`LinkedList`] +- the growable ring buffer [`VecDeque`] +- the [`BinaryHeap`] priority queue +- [`BTreeMap`] and [`BTreeSet`] + +[`Arc`]: https://doc.rust-lang.org/stable/alloc/sync/struct.Arc.html +[`String`]: https://doc.rust-lang.org/collections/string/struct.String.html +[`format!`]: https://doc.rust-lang.org/alloc/macro.format.html +[`LinkedList`]: https://doc.rust-lang.org/collections/linked_list/struct.LinkedList.html +[`VecDeque`]: https://doc.rust-lang.org/collections/vec_deque/struct.VecDeque.html +[`BinaryHeap`]: https://doc.rust-lang.org/collections/binary_heap/struct.BinaryHeap.html +[`BTreeMap`]: https://doc.rust-lang.org/collections/btree_map/struct.BTreeMap.html +[`BTreeSet`]: https://doc.rust-lang.org/collections/btree_set/struct.BTreeSet.html + +These types will become very useful when we want to implement thread lists, scheduling queues, or support for async/await. ## Summary +This post gave an introduction to dynamic memory and explained why and where it is needed. We saw how Rust's borrow checker prevents common vulnerabilities and learned how Rust's allocation API works. + +After creating a minimal implementation of Rust's allocator interface using a dummy allocator, we created a proper heap memory region for our kernel. For that we defined a virtual address range for the heap and then mapped all pages of that range to physical frames using the `Mapper` and `FrameAllocator` from the previous post. + +Finally, we added a dependency on the `linked_list_allocator` crate to add a proper allocator to our kernel. With this allocator, we were able to use `Box`, `Vec`, and other allocation and collection types from the `alloc` crate. + ## What's next? -allocator designs (optional) \ No newline at end of file + +While we already added heap allocation support in this post, we left most of the work to the `linked_list_allocator` crate. The next post will show in detail how an allocator can be implemented from scratch. It will present multiple possible allocator designs, shows how to implement simple versions of them, and explain their advantages and drawbacks. + +TODO: +- update date +- resolve todos +- check spelling +- create post-10 tag diff --git a/blog/content/second-edition/posts/10-heap-allocation/qemu-alloc-showcase.png b/blog/content/second-edition/posts/10-heap-allocation/qemu-alloc-showcase.png new file mode 100644 index 0000000000000000000000000000000000000000..a849aa7bf010fe976bd6be6a4bafd895e84b2f1b GIT binary patch literal 9485 zcmeAS@N?(olHy`uVBq!ia0y~yV7kP>z_^x!je&u|`rBk>1_lO}VkgfK4h{~E8jh3> z1_lO+64!{5;QX|b^2DN4hTO!GRNdm_qSVy9;*9)~6VpC;F)%1Fc)B=-RLpsMw|YX# z^+fq+_upRMQzp(WpmKoK^`MGq;DgyB_ct(jv6XLD)D4#rSB{*dJ@o_s>B1*2VLOGU z%V_&qE-jjx<2rSdQ1qIJX{y1Kbfd%^8w^?PR0ECVpUs})Fp=qF6w~i}m8IYJ{#=}% zmY)9nVsTnodD`>et7hGKY4`n3vH$m^`r2oAidU_9d~f@{jgjvw85k@&?EZE7hh@*- z{iyRPL+zWN~KgfM+Xye9yeQle#->YTi z_Wx?o?!WQxV|M+WGT!hjfmder?oR%*Yt`Gh-=X4hMNh-8Y&{ykc5U{}C-1M`SbO7- zy=t`l{+Ei$`?s-BZhv~VO8&I_q-j-o?aqPI`}J36oIRd+yL4^+pC4ay_y3fxDdX1LANl<6 zoqHd%=ij)vJoD`2*7E%=I^3{pfG= zF=Uc9^Vd`I?`mI5&yLQw-4!a#!0_OJRq?YoUNhEB3hc0}`Z7)WwiW}!4fb>MY^V3v zKbjo++RdDWf#KWYSD-N5`E;7H`7T)ohBwK+mzS(sb)|Ifu9d<3vrSe`U#4d6xhmx9 zr@wZgv(o+k4o`=(WC5Tbrf1VcYGOWw$-AN}biu+a%OA{q@QqRo}_W zR$hM3e|$xH$i7vpwp=^C#rv()mBUxQK0RVsL?%KRLkD1|t zLNw>aW~;ydlElQsi@)D3f4**a-&4!;J8ie$t1?MHH|Ol;^LCSu%hxO2|8dOR=;G_8 zm*4idP1$U-$wn$FZdvxc| zGkQFiRO|KqOylb4Q@{%B6CAsAG+h_Nx?{5k6Uwt)dmei&5_p3jbFDrgoA}03C zY=4LM^UHJRoV#CtHn#G+`o4K(yXS^w@9BuC*_^ol{`sB7;%aJ%1=-=ADQd*wyP!|ILh`NGNi?Nfb`w|Qx5wx3&&RG+^SylPVsyXC;o$JMC6`Uk zzKFhd>zLWuzVa)Zsp;J0a@o_{=i5z<+dVz8eunk?J;v>6Y13YL%2hs5mgHf3J@0oTd!54Hy85N_ zf|vWHF0!3?Fk!;v@^eor&*uC}Kd;xL>h5OI{{TdLpX- zd)}WH{eL|E9o_$Ds?N9HGiT5CzxidNHh1=G>ma+c!$L8me$uG?oYbqV@ zyF2se&p_ML?RFmye2))3uiRf_^0%%&bLJi^{T;6(=G8np86q6Yb~y1}^}7(Z>F1Wq z^-Owj`SamFcb3bY_TTsG=g#NbUe>&Q6Z7w4`n(hM|8I#uJ9jd0=E0?(cmCfQ{PV!& z;+MTMpHDlscKiH)ZC?%c=BeM!Q(s^A^mNg`J3qJFTHF^oWB&Qv;(vDW_wKj}ugcx47cBq% z>Ce~y&uxFcKF#_%{qqC+AM-!2?)P7E+2`4biFvD@IoW;R*Z=&)_kR)p4xJ2tQeOA* zbmhy=-^Je_iGT9H|NZHnpS<~}%I_6f%v~0D`IFk~<1cbP$S@o@yK>c}Wpd9hobeG8 z7oQYkdpWVv>)6xh?JHMl9e$`_eEIU?+5e4JrKhE-b$54%UokW{FL$*)?e?>%B>&!* zAltQDg7{z898a10>r&3FU4E1H{Hb*R9Q=Nd{{I*E>n-YkzFYirZT_6o{&W6(u5>N6 z``T*kIcb;NGgjl^mF=a+&Xybvw!XDCENZp*;kepESwFw{TFw17>)h?OWu9tkMl+Yp z+&$&y(&u5d>Yo1}<%mtb=-Yc*|H}Di#r-u_{ks17d3sN^#c#a~DSah#^5IOavxk<( z*nDOGIrskW?w`BA+fDdwSE91M?z3*?vt<6~-{;RdrOobXTC8F0Ta@^<=5yLk-m_M< zIa%xs1qCLauac`}`7mSqo~OOqi?0e#etX6H=DIkpQ{b%jnki}n@FQ5I-YlHu!UvIX5zT5u$=lXjey8i5!|DRL$M0fp})2}W| z&atoEnHW0t8mce8YKbb6NcZi!mCDopQ;Zc0+b^2}FLKgIvtto|u~f03U3pJ(CE zYX5)q-uZiX{>krmiYiNYuDW{a=+2)Zq1kV2wr^=E+WhwTmoFi^?)u$&tM&hn?faAI z_TP{C_@7Uys#2T(_s{fC`gK32U(VZp>2;}*scB$I-O;5#&{O-JeM|Z~T zv6y~PHvdd?+$BHb-xd3Q`drErjal3d0mtp?wnVL+V_9r=IC1X5gb5D|CfvMv^UT?^ zsd;&NpFe-ruAet|@05oH6CNf^c(~!*&CSm*Woug;I!QzM*w zDaAlSZ|?eYJBy!B-}Y-+S@#pushg8i3WfBWou{b%z} z@9+Qrx5P8;dD`XIUxTjuzT5HmoQ}?!=YM{FzHR3owDfwVj(pXN)s`Rc?=i9e@%VqK z+;8>$6YXub&e+oTWa;}K?>{YGKilu~TlY^N(w|*AcWIUE+vJ_^*Z<3i+f(OOok_NEbF50W?v|;q-Lh^~ z`j%}apV;d^xWArPZSZ3Gy4h2!tmfveS~fLq`}BU>YDN9(yh}R1<}>HbI`{vBc-*Y_ zkv3ERZJ&R(n_vFyeY=0leCGVRY|?x)u=Q=6TxHsXxbGK4m2b}0oOi46-JPF4zPz7K zh3~T}E-sejVSicbe)w_s-=kZi-8`oY{)B&CU;ps_lhrR{oPtZul!Rk zcTRlQlf8utFiw_y=mt6s+k$)kr`s{%9r+@^w~PgIt9y>n;C3Y<_=wz3rzIwPjw**E2A@@h`o*moE#Kcv;QWel5u} zd(W?{>%Dg0jhgk$@YdfA>)aP#+FV?m85*j0_T00Z>GRHB`BD<3Dm+>E=ZBj&=h$32 z&%G+_)xo4bU*5?#)x(~-@SnGvEY9EVx!%v#+GF+Y)64bDex5j5)_2cp;?JGB*XG3q zfjeJ&eAmc}jHlsx46~+kNF~zxZEzS90mK-;%3Rx$p1B z%jmBEvq=2!&XBb|VP7Ib{U{O{b>zttQFEbdrIU}gw(_)>_Q~!x4h|0XE?x6)YyH+O zHTMke|GM^gf9o~*>36eNg=x>t`*cR!ZoYc@ zyK~w1zD&FGecSe@N3_L0f8?)Ioo?mU#>lX4`LfI){`Q$KzE0AzEdJ#qCMGs{mFMca zUXw3Mo!z{7bLF?2mn*;BOuzivZ;83#MxjNkv$m8Kg@$^Y%bQu&{yLN4xmPQ5Yt+iu zOY&Zw(LeM2&CJb}uU0O<{BDc!#n(%tPHlSYr@mNIA#~T>IJUKZc2x`K9v7R`x$30P z)s#h3V^&1}4B9{ad)$nZ`)BS}-Rtar>GfA5YjgW=bDr;7=dOHd>Fx6S7QVl@n{~G3 zoh;w;qIKuuI-B|bH-CP)pKo&3p!=^Q+uwtiO8TrquX?S%>b3l`-}dXitFvMSkL}*ux%yVv zbzd>DXPbBInD8(mBz*N%DPQfEz9~sbM$P|{#KgrerR`pn-W-{+}) z-QXD;`f^K421A4N(vwf8cHDk@E~BJUb$%@0?$fWm^S9W2`Sf|Q>7vPNZl+dkaXpc7 zrE<%r<4>!mJ@?4Jd2N^9-7At`+9M-kB5pDkTDKie{Q0T8+V$$HOXvRWV`FHTeXvMM zI;W`k{#Fxf>+0C^*CqeX7vN%GxE;8s&xCZSAps-@&}n1JU{u z`izyPs6 zH|Onc^nP6)&c6R{#PzB<$^1z(3xEBrn#RabcJ4|abI#t2m*=vq-zyaLJqj-2p1xPz z{MXknRsRL@L(>*tU;S#Y^(?=Qq0;7D{Hn_5rLRU_vcGM4->o#qjA`fXZMV;7-@o45 zXPJ@8&@lUasPZN0+}OR}t{Y)l+Y8&*nWov*^IP$o2CX8TdBIvc8nIWiKwBaj#qPSJgNFlRJ<7HQH46@B6Lf z8+V}Lw|lWw<)ezv|M+G0{yCy%J|VaKYG3a%<~P?~iDkdOoptYB+GXKwn}d#R-M6mr z_MW(#LbGFk?^s>d6MWrKfAjQPObl<%oD@Fp?)VGT@>!>D=ETNupswuGk&kch=Tz9{ znC3b#uroYx+x^b5>g}DgU!68C;9)orSAMs7%LE>V17{QN>(#R}FqlsQH;KMIIoo#q zP+rc>J-5BPdN)^{bkx~=H}#3vYsu|A3un?vo_*VZon zo;2tCX*Xkr1Aa0c3VVZJ_s)B#Qt@``guZ}gT~*k^c9xxxc%;3 zyY$qXezT9+p5J8=WW}t?M>Ka zJHD!!SIb@-oA}u>Sug}7y;&*y*e5pO?>|O{2hWN_C)~@~_u|&im8<99J`p#2aa7T| z!u>80e!1youTS5y`1&J>gW*j`>8B$fLG91D%B1DyeRo%cTAR%d+6aoVwL(VkR>?lT zQhMQZ&iAj`cjFiuw))Mps4Jbo#=u}!{^>}e^d<%d2TrWb&>d>l((hcWGr3>hjp*us zoxR=8{(AD-S9hc0gcxqzX_T0F&wKWwJ1f?n)!aV+-r}vZnSRb;Jg~Jm!EMR+MOSB^ zC@_e5Juy-FJVU|XH7qaJ*2(U<=<&)e)oj*21HYMXx23Nx&Mo16d~2=Q`sFF*Ctlxd ze|*I_Z)V|}V>`=!Z{8KS_x9GSQ|4tGsvh3^gSSsMdJ*WZ%8j3=Aw&Kx+X? z*Ihb)bnBl#Iic6{zCN^g&csmm4%Alne|zJl{l3RC`$K2$wRoGTo_p^__tCP~H`jif za_hyM>z`xHXFolE?A2_k^2gox9beir>=0b{Q|9GdmREaanAVlf-X48E?#1D~f9l?{ zuRi?c(|OgF42B(cd`DKvZis(VvXoa*iecS+-sUf_CT`rDu>DrC?fUHXTYpwv+kE}j zp6GR<)z_}=o_+e-+pTsi4YOTgk$LZ17U%4{54K(W_;_06`a5ZFZ@x;~`1RI4+1*#4 z?F~Es%INW(Gv;`~YB z<7X$bF*FEa?THllHhuZDYwNXjckRnw@4eOkx`dfw%8joId#~QQ^v3r0N84Ndxoix1 zde=*`+IbcD=3R}BiPv^AVb~yEGD+KgqWSHEYg3+YEi5ySiO$=8YxC^uTc?-3yL~RN zWp?cETc2~bZN6`@?bhG??Y|1l>uwk7+9gY0$11Mxt7k%V6nq0d;SeA zk6baOu>N;%m(07H<-fA`T&3;#(#?_oa&*|QKYh-?@NM<#rZ1a%ozCU8&VPHYc?0s=RpO`)4$lu0wq<39<>CD2g9TLmu72eu9^RCtPTLHVS zo|1k2cTZ%k>b9!;>r&;XUDumGsy8>_b#(SIJPLsL)GJ5tT1_pM|^w&z}*W$QNGB9k&d84qmtfg@I{zLNVM~bY2 z?Y-9S>pOdH``x<|<(KRq&&%GOdoRoV@Yz#q%kvl-nseUF=3V`2Z@BN@8<`#VUj0}b zcfU7adsX!FtJk;OGiiN#{krz%=)cDAc3ztn8=bb=>X_~ITc2i6l}yhPU^sC0(JueD zH`WTCZ8kC5b})4sATm75<=JH6XcfFYr# z$t2nLwZ=+rh6ice7Axe66c_KlYJYvz{Oi|tODEn+XJW9>SjTbc@fzkm7kl#_M_Erz z*!*EzWc9Y-^L}e>ucuw#cl+<=INRH`^UYq(e>;zxq2ctZ$?ARP{B~bH^yaZMFuajg z_m?qumttV>qO`w#Ak=YJ=Dn%QDz7hN*DN+T&A?DFQxMdr4*;i<4-4D#ORO0f67E}6 sKH346P(*b0OD0{E;b&k-sQGU{nZve8y1Qd90|Nttr>mdKI;Vst0Mydvp#T5? literal 0 HcmV?d00001