Annotation of funnyos/libkern/memcpy.S, Revision 1.1.1.1
1.1 init 1: /* $Id: memcpy.S,v 1.1.1.1 2007/10/12 08:40:43 init Exp $ */
2: /* $OpenBSD: memcpy.S,v 1.2 2004/02/01 05:47:10 drahn Exp $ */
3: /* $NetBSD: memcpy.S,v 1.2 2001/11/20 00:29:20 chris Exp $ */
4:
5: /*-
6: * Copyright (c) 1997 The NetBSD Foundation, Inc.
7: * All rights reserved.
8: *
9: * This code is derived from software contributed to The NetBSD Foundation
10: * by Neil A. Carson and Mark Brinicombe
11: *
12: * Redistribution and use in source and binary forms, with or without
13: * modification, are permitted provided that the following conditions
14: * are met:
15: * 1. Redistributions of source code must retain the above copyright
16: * notice, this list of conditions and the following disclaimer.
17: * 2. Redistributions in binary form must reproduce the above copyright
18: * notice, this list of conditions and the following disclaimer in the
19: * documentation and/or other materials provided with the distribution.
20: * 3. All advertising materials mentioning features or use of this software
21: * must display the following acknowledgement:
22: * This product includes software developed by the NetBSD
23: * Foundation, Inc. and its contributors.
24: * 4. Neither the name of The NetBSD Foundation nor the names of its
25: * contributors may be used to endorse or promote products derived
26: * from this software without specific prior written permission.
27: *
28: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
29: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
30: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
31: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
32: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38: * POSSIBILITY OF SUCH DAMAGE.
39: */
40:
41: #include <libkern/asm.h>
42:
43: /*
44: * This is one fun bit of code ...
45: * Some easy listening music is suggested while trying to understand this
46: * code e.g. Iron Maiden
47: *
48: * For anyone attempting to understand it :
49: *
50: * The core code is implemented here with simple stubs for memcpy()
51: * memmove() and bcopy().
52: *
53: * All local labels are prefixed with Lmemcpy_
54: * Following the prefix a label starting f is used in the forward copy code
55: * while a label using b is used in the backwards copy code
56: * The source and destination addresses determine whether a forward or
57: * backward copy is performed.
58: * Separate bits of code are used to deal with the following situations
59: * for both the forward and backwards copy.
60: * unaligned source address
61: * unaligned destination address
62: * Separate copy routines are used to produce an optimised result for each
63: * of these cases.
64: * The copy code will use LDM/STM instructions to copy up to 32 bytes at
65: * a time where possible.
66: *
67: * Note: r12 (aka ip) can be trashed during the function along with
68: * r0-r3 although r0-r2 have defined uses i.e. src, dest, len through out.
69: * Additional registers are preserved prior to use i.e. r4, r5 & lr
70: *
71: * Apologies for the state of the comments ;-)
72: */
73:
74: ENTRY(memcpy)
75: ENTRY_NP(memmove)
76: /* Determine copy direction */
77: cmp r1, r0
78:
79: moveq r0, #0 /* Quick abort for len=0 */
80: #ifdef __APCS_26__
81: moveqs pc, lr
82: #else
83: moveq pc, lr
84: #endif
85:
86: /* save leaf functions having to store this away */
87: stmdb sp!, {r0, lr} /* memcpy() returns dest addr */
88:
89: bcc Lmemcpy_backwards
90:
91: /* start of forwards copy */
92: subs r2, r2, #4
93: blt Lmemcpy_fl4 /* less than 4 bytes */
94: ands r12, r0, #3
95: bne Lmemcpy_fdestul /* oh unaligned destination addr */
96: ands r12, r1, #3
97: bne Lmemcpy_fsrcul /* oh unaligned source addr */
98:
99: Lmemcpy_ft8:
100: /* We have aligned source and destination */
101: subs r2, r2, #8
102: blt Lmemcpy_fl12 /* less than 12 bytes (4 from above) */
103: subs r2, r2, #0x14
104: blt Lmemcpy_fl32 /* less than 32 bytes (12 from above) */
105: stmdb sp!, {r4} /* borrow r4 */
106:
107: /* blat 32 bytes at a time */
108: /* XXX for really big copies perhaps we should use more registers */
109: Lmemcpy_floop32:
110: ldmia r1!, {r3, r4, r12, lr}
111: stmia r0!, {r3, r4, r12, lr}
112: ldmia r1!, {r3, r4, r12, lr}
113: stmia r0!, {r3, r4, r12, lr}
114: subs r2, r2, #0x20
115: bge Lmemcpy_floop32
116:
117: cmn r2, #0x10
118: ldmgeia r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */
119: stmgeia r0!, {r3, r4, r12, lr}
120: subge r2, r2, #0x10
121: ldmia sp!, {r4} /* return r4 */
122:
123: Lmemcpy_fl32:
124: adds r2, r2, #0x14
125:
126: /* blat 12 bytes at a time */
127: Lmemcpy_floop12:
128: ldmgeia r1!, {r3, r12, lr}
129: stmgeia r0!, {r3, r12, lr}
130: subges r2, r2, #0x0c
131: bge Lmemcpy_floop12
132:
133: Lmemcpy_fl12:
134: adds r2, r2, #8
135: blt Lmemcpy_fl4
136:
137: subs r2, r2, #4
138: ldrlt r3, [r1], #4
139: strlt r3, [r0], #4
140: ldmgeia r1!, {r3, r12}
141: stmgeia r0!, {r3, r12}
142: subge r2, r2, #4
143:
144: Lmemcpy_fl4:
145: /* less than 4 bytes to go */
146: adds r2, r2, #4
147: #ifdef __APCS_26_
148: ldmeqia sp!, {r0, pc}^ /* done */
149: #else
150: ldmeqia sp!, {r0, pc} /* done */
151: #endif
152: /* copy the crud byte at a time */
153: cmp r2, #2
154: ldrb r3, [r1], #1
155: strb r3, [r0], #1
156: ldrgeb r3, [r1], #1
157: strgeb r3, [r0], #1
158: ldrgtb r3, [r1], #1
159: strgtb r3, [r0], #1
160: #ifdef __APCS_26__
161: ldmia sp!, {r0, pc}^
162: #else
163: ldmia sp!, {r0, pc}
164: #endif
165:
166: /* erg - unaligned destination */
167: Lmemcpy_fdestul:
168: rsb r12, r12, #4
169: cmp r12, #2
170:
171: /* align destination with byte copies */
172: ldrb r3, [r1], #1
173: strb r3, [r0], #1
174: ldrgeb r3, [r1], #1
175: strgeb r3, [r0], #1
176: ldrgtb r3, [r1], #1
177: strgtb r3, [r0], #1
178: subs r2, r2, r12
179: blt Lmemcpy_fl4 /* less the 4 bytes */
180:
181: ands r12, r1, #3
182: beq Lmemcpy_ft8 /* we have an aligned source */
183:
184: /* erg - unaligned source */
185: /* This is where it gets nasty ... */
186: Lmemcpy_fsrcul:
187: bic r1, r1, #3
188: ldr lr, [r1], #4
189: cmp r12, #2
190: bgt Lmemcpy_fsrcul3
191: beq Lmemcpy_fsrcul2
192: cmp r2, #0x0c
193: blt Lmemcpy_fsrcul1loop4
194: sub r2, r2, #0x0c
195: stmdb sp!, {r4, r5}
196:
197: Lmemcpy_fsrcul1loop16:
198: mov r3, lr, lsr #8
199: ldmia r1!, {r4, r5, r12, lr}
200: orr r3, r3, r4, lsl #24
201: mov r4, r4, lsr #8
202: orr r4, r4, r5, lsl #24
203: mov r5, r5, lsr #8
204: orr r5, r5, r12, lsl #24
205: mov r12, r12, lsr #8
206: orr r12, r12, lr, lsl #24
207: stmia r0!, {r3-r5, r12}
208: subs r2, r2, #0x10
209: bge Lmemcpy_fsrcul1loop16
210: ldmia sp!, {r4, r5}
211: adds r2, r2, #0x0c
212: blt Lmemcpy_fsrcul1l4
213:
214: Lmemcpy_fsrcul1loop4:
215: mov r12, lr, lsr #8
216: ldr lr, [r1], #4
217: orr r12, r12, lr, lsl #24
218: str r12, [r0], #4
219: subs r2, r2, #4
220: bge Lmemcpy_fsrcul1loop4
221:
222: Lmemcpy_fsrcul1l4:
223: sub r1, r1, #3
224: b Lmemcpy_fl4
225:
226: Lmemcpy_fsrcul2:
227: cmp r2, #0x0c
228: blt Lmemcpy_fsrcul2loop4
229: sub r2, r2, #0x0c
230: stmdb sp!, {r4, r5}
231:
232: Lmemcpy_fsrcul2loop16:
233: mov r3, lr, lsr #16
234: ldmia r1!, {r4, r5, r12, lr}
235: orr r3, r3, r4, lsl #16
236: mov r4, r4, lsr #16
237: orr r4, r4, r5, lsl #16
238: mov r5, r5, lsr #16
239: orr r5, r5, r12, lsl #16
240: mov r12, r12, lsr #16
241: orr r12, r12, lr, lsl #16
242: stmia r0!, {r3-r5, r12}
243: subs r2, r2, #0x10
244: bge Lmemcpy_fsrcul2loop16
245: ldmia sp!, {r4, r5}
246: adds r2, r2, #0x0c
247: blt Lmemcpy_fsrcul2l4
248:
249: Lmemcpy_fsrcul2loop4:
250: mov r12, lr, lsr #16
251: ldr lr, [r1], #4
252: orr r12, r12, lr, lsl #16
253: str r12, [r0], #4
254: subs r2, r2, #4
255: bge Lmemcpy_fsrcul2loop4
256:
257: Lmemcpy_fsrcul2l4:
258: sub r1, r1, #2
259: b Lmemcpy_fl4
260:
261: Lmemcpy_fsrcul3:
262: cmp r2, #0x0c
263: blt Lmemcpy_fsrcul3loop4
264: sub r2, r2, #0x0c
265: stmdb sp!, {r4, r5}
266:
267: Lmemcpy_fsrcul3loop16:
268: mov r3, lr, lsr #24
269: ldmia r1!, {r4, r5, r12, lr}
270: orr r3, r3, r4, lsl #8
271: mov r4, r4, lsr #24
272: orr r4, r4, r5, lsl #8
273: mov r5, r5, lsr #24
274: orr r5, r5, r12, lsl #8
275: mov r12, r12, lsr #24
276: orr r12, r12, lr, lsl #8
277: stmia r0!, {r3-r5, r12}
278: subs r2, r2, #0x10
279: bge Lmemcpy_fsrcul3loop16
280: ldmia sp!, {r4, r5}
281: adds r2, r2, #0x0c
282: blt Lmemcpy_fsrcul3l4
283:
284: Lmemcpy_fsrcul3loop4:
285: mov r12, lr, lsr #24
286: ldr lr, [r1], #4
287: orr r12, r12, lr, lsl #8
288: str r12, [r0], #4
289: subs r2, r2, #4
290: bge Lmemcpy_fsrcul3loop4
291:
292: Lmemcpy_fsrcul3l4:
293: sub r1, r1, #1
294: b Lmemcpy_fl4
295:
296: Lmemcpy_backwards:
297: add r1, r1, r2
298: add r0, r0, r2
299: subs r2, r2, #4
300: blt Lmemcpy_bl4 /* less than 4 bytes */
301: ands r12, r0, #3
302: bne Lmemcpy_bdestul /* oh unaligned destination addr */
303: ands r12, r1, #3
304: bne Lmemcpy_bsrcul /* oh unaligned source addr */
305:
306: Lmemcpy_bt8:
307: /* We have aligned source and destination */
308: subs r2, r2, #8
309: blt Lmemcpy_bl12 /* less than 12 bytes (4 from above) */
310: stmdb sp!, {r4}
311: subs r2, r2, #0x14 /* less than 32 bytes (12 from above) */
312: blt Lmemcpy_bl32
313:
314: /* blat 32 bytes at a time */
315: /* XXX for really big copies perhaps we should use more registers */
316: Lmemcpy_bloop32:
317: ldmdb r1!, {r3, r4, r12, lr}
318: stmdb r0!, {r3, r4, r12, lr}
319: ldmdb r1!, {r3, r4, r12, lr}
320: stmdb r0!, {r3, r4, r12, lr}
321: subs r2, r2, #0x20
322: bge Lmemcpy_bloop32
323:
324: Lmemcpy_bl32:
325: cmn r2, #0x10
326: ldmgedb r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */
327: stmgedb r0!, {r3, r4, r12, lr}
328: subge r2, r2, #0x10
329: adds r2, r2, #0x14
330: ldmgedb r1!, {r3, r12, lr} /* blat a remaining 12 bytes */
331: stmgedb r0!, {r3, r12, lr}
332: subge r2, r2, #0x0c
333: ldmia sp!, {r4}
334:
335: Lmemcpy_bl12:
336: adds r2, r2, #8
337: blt Lmemcpy_bl4
338: subs r2, r2, #4
339: ldrlt r3, [r1, #-4]!
340: strlt r3, [r0, #-4]!
341: ldmgedb r1!, {r3, r12}
342: stmgedb r0!, {r3, r12}
343: subge r2, r2, #4
344:
345: Lmemcpy_bl4:
346: /* less than 4 bytes to go */
347: adds r2, r2, #4
348: #ifdef __APCS_26__
349: ldmeqia sp!, {r0, pc}^
350: #else
351: ldmeqia sp!, {r0, pc}
352: #endif
353:
354: /* copy the crud byte at a time */
355: cmp r2, #2
356: ldrb r3, [r1, #-1]!
357: strb r3, [r0, #-1]!
358: ldrgeb r3, [r1, #-1]!
359: strgeb r3, [r0, #-1]!
360: ldrgtb r3, [r1, #-1]!
361: strgtb r3, [r0, #-1]!
362: #ifdef __APCS_26__
363: ldmia sp!, {r0, pc}^
364: #else
365: ldmia sp!, {r0, pc}
366: #endif
367:
368: /* erg - unaligned destination */
369: Lmemcpy_bdestul:
370: cmp r12, #2
371:
372: /* align destination with byte copies */
373: ldrb r3, [r1, #-1]!
374: strb r3, [r0, #-1]!
375: ldrgeb r3, [r1, #-1]!
376: strgeb r3, [r0, #-1]!
377: ldrgtb r3, [r1, #-1]!
378: strgtb r3, [r0, #-1]!
379: subs r2, r2, r12
380: blt Lmemcpy_bl4 /* less than 4 bytes to go */
381: ands r12, r1, #3
382: beq Lmemcpy_bt8 /* we have an aligned source */
383:
384: /* erg - unaligned source */
385: /* This is where it gets nasty ... */
386: Lmemcpy_bsrcul:
387: bic r1, r1, #3
388: ldr r3, [r1, #0]
389: cmp r12, #2
390: blt Lmemcpy_bsrcul1
391: beq Lmemcpy_bsrcul2
392: cmp r2, #0x0c
393: blt Lmemcpy_bsrcul3loop4
394: sub r2, r2, #0x0c
395: stmdb sp!, {r4, r5}
396:
397: Lmemcpy_bsrcul3loop16:
398: mov lr, r3, lsl #8
399: ldmdb r1!, {r3-r5, r12}
400: orr lr, lr, r12, lsr #24
401: mov r12, r12, lsl #8
402: orr r12, r12, r5, lsr #24
403: mov r5, r5, lsl #8
404: orr r5, r5, r4, lsr #24
405: mov r4, r4, lsl #8
406: orr r4, r4, r3, lsr #24
407: stmdb r0!, {r4, r5, r12, lr}
408: subs r2, r2, #0x10
409: bge Lmemcpy_bsrcul3loop16
410: ldmia sp!, {r4, r5}
411: adds r2, r2, #0x0c
412: blt Lmemcpy_bsrcul3l4
413:
414: Lmemcpy_bsrcul3loop4:
415: mov r12, r3, lsl #8
416: ldr r3, [r1, #-4]!
417: orr r12, r12, r3, lsr #24
418: str r12, [r0, #-4]!
419: subs r2, r2, #4
420: bge Lmemcpy_bsrcul3loop4
421:
422: Lmemcpy_bsrcul3l4:
423: add r1, r1, #3
424: b Lmemcpy_bl4
425:
426: Lmemcpy_bsrcul2:
427: cmp r2, #0x0c
428: blt Lmemcpy_bsrcul2loop4
429: sub r2, r2, #0x0c
430: stmdb sp!, {r4, r5}
431:
432: Lmemcpy_bsrcul2loop16:
433: mov lr, r3, lsl #16
434: ldmdb r1!, {r3-r5, r12}
435: orr lr, lr, r12, lsr #16
436: mov r12, r12, lsl #16
437: orr r12, r12, r5, lsr #16
438: mov r5, r5, lsl #16
439: orr r5, r5, r4, lsr #16
440: mov r4, r4, lsl #16
441: orr r4, r4, r3, lsr #16
442: stmdb r0!, {r4, r5, r12, lr}
443: subs r2, r2, #0x10
444: bge Lmemcpy_bsrcul2loop16
445: ldmia sp!, {r4, r5}
446: adds r2, r2, #0x0c
447: blt Lmemcpy_bsrcul2l4
448:
449: Lmemcpy_bsrcul2loop4:
450: mov r12, r3, lsl #16
451: ldr r3, [r1, #-4]!
452: orr r12, r12, r3, lsr #16
453: str r12, [r0, #-4]!
454: subs r2, r2, #4
455: bge Lmemcpy_bsrcul2loop4
456:
457: Lmemcpy_bsrcul2l4:
458: add r1, r1, #2
459: b Lmemcpy_bl4
460:
461: Lmemcpy_bsrcul1:
462: cmp r2, #0x0c
463: blt Lmemcpy_bsrcul1loop4
464: sub r2, r2, #0x0c
465: stmdb sp!, {r4, r5}
466:
467: Lmemcpy_bsrcul1loop32:
468: mov lr, r3, lsl #24
469: ldmdb r1!, {r3-r5, r12}
470: orr lr, lr, r12, lsr #8
471: mov r12, r12, lsl #24
472: orr r12, r12, r5, lsr #8
473: mov r5, r5, lsl #24
474: orr r5, r5, r4, lsr #8
475: mov r4, r4, lsl #24
476: orr r4, r4, r3, lsr #8
477: stmdb r0!, {r4, r5, r12, lr}
478: subs r2, r2, #0x10
479: bge Lmemcpy_bsrcul1loop32
480: ldmia sp!, {r4, r5}
481: adds r2, r2, #0x0c
482: blt Lmemcpy_bsrcul1l4
483:
484: Lmemcpy_bsrcul1loop4:
485: mov r12, r3, lsl #24
486: ldr r3, [r1, #-4]!
487: orr r12, r12, r3, lsr #8
488: str r12, [r0, #-4]!
489: subs r2, r2, #4
490: bge Lmemcpy_bsrcul1loop4
491:
492: Lmemcpy_bsrcul1l4:
493: add r1, r1, #1
494: b Lmemcpy_bl4
495:
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to memcpy.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / funnyos / libkern