Annotation of sys/lib/libkern/arch/arm/memcpy.S, Revision 1.1
1.1 ! nbrk 1: /* $OpenBSD: memcpy.S,v 1.2 2004/02/01 05:47:10 drahn Exp $ */
! 2: /* $NetBSD: memcpy.S,v 1.2 2001/11/20 00:29:20 chris Exp $ */
! 3:
! 4: /*-
! 5: * Copyright (c) 1997 The NetBSD Foundation, Inc.
! 6: * All rights reserved.
! 7: *
! 8: * This code is derived from software contributed to The NetBSD Foundation
! 9: * by Neil A. Carson and Mark Brinicombe
! 10: *
! 11: * Redistribution and use in source and binary forms, with or without
! 12: * modification, are permitted provided that the following conditions
! 13: * are met:
! 14: * 1. Redistributions of source code must retain the above copyright
! 15: * notice, this list of conditions and the following disclaimer.
! 16: * 2. Redistributions in binary form must reproduce the above copyright
! 17: * notice, this list of conditions and the following disclaimer in the
! 18: * documentation and/or other materials provided with the distribution.
! 19: * 3. All advertising materials mentioning features or use of this software
! 20: * must display the following acknowledgement:
! 21: * This product includes software developed by the NetBSD
! 22: * Foundation, Inc. and its contributors.
! 23: * 4. Neither the name of The NetBSD Foundation nor the names of its
! 24: * contributors may be used to endorse or promote products derived
! 25: * from this software without specific prior written permission.
! 26: *
! 27: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
! 28: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
! 29: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
! 30: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
! 31: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
! 32: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
! 33: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
! 34: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
! 35: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
! 36: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
! 37: * POSSIBILITY OF SUCH DAMAGE.
! 38: */
! 39:
! 40: #include <machine/asm.h>
! 41:
! 42: /*
! 43: * This is one fun bit of code ...
! 44: * Some easy listening music is suggested while trying to understand this
! 45: * code e.g. Iron Maiden
! 46: *
! 47: * For anyone attempting to understand it :
! 48: *
! 49: * The core code is implemented here with simple stubs for memcpy()
! 50: * memmove() and bcopy().
! 51: *
! 52: * All local labels are prefixed with Lmemcpy_
! 53: * Following the prefix a label starting f is used in the forward copy code
! 54: * while a label using b is used in the backwards copy code
! 55: * The source and destination addresses determine whether a forward or
! 56: * backward copy is performed.
! 57: * Separate bits of code are used to deal with the following situations
! 58: * for both the forward and backwards copy.
! 59: * unaligned source address
! 60: * unaligned destination address
! 61: * Separate copy routines are used to produce an optimised result for each
! 62: * of these cases.
! 63: * The copy code will use LDM/STM instructions to copy up to 32 bytes at
! 64: * a time where possible.
! 65: *
! 66: * Note: r12 (aka ip) can be trashed during the function along with
! 67: * r0-r3 although r0-r2 have defined uses i.e. src, dest, len through out.
! 68: * Additional registers are preserved prior to use i.e. r4, r5 & lr
! 69: *
! 70: * Apologies for the state of the comments ;-)
! 71: */
! 72:
! 73: ENTRY(memcpy)
! 74: ENTRY_NP(memmove)
! 75: /* Determine copy direction */
! 76: cmp r1, r0
! 77:
! 78: moveq r0, #0 /* Quick abort for len=0 */
! 79: #ifdef __APCS_26__
! 80: moveqs pc, lr
! 81: #else
! 82: moveq pc, lr
! 83: #endif
! 84:
! 85: /* save leaf functions having to store this away */
! 86: stmdb sp!, {r0, lr} /* memcpy() returns dest addr */
! 87:
! 88: bcc Lmemcpy_backwards
! 89:
! 90: /* start of forwards copy */
! 91: subs r2, r2, #4
! 92: blt Lmemcpy_fl4 /* less than 4 bytes */
! 93: ands r12, r0, #3
! 94: bne Lmemcpy_fdestul /* oh unaligned destination addr */
! 95: ands r12, r1, #3
! 96: bne Lmemcpy_fsrcul /* oh unaligned source addr */
! 97:
! 98: Lmemcpy_ft8:
! 99: /* We have aligned source and destination */
! 100: subs r2, r2, #8
! 101: blt Lmemcpy_fl12 /* less than 12 bytes (4 from above) */
! 102: subs r2, r2, #0x14
! 103: blt Lmemcpy_fl32 /* less than 32 bytes (12 from above) */
! 104: stmdb sp!, {r4} /* borrow r4 */
! 105:
! 106: /* blat 32 bytes at a time */
! 107: /* XXX for really big copies perhaps we should use more registers */
! 108: Lmemcpy_floop32:
! 109: ldmia r1!, {r3, r4, r12, lr}
! 110: stmia r0!, {r3, r4, r12, lr}
! 111: ldmia r1!, {r3, r4, r12, lr}
! 112: stmia r0!, {r3, r4, r12, lr}
! 113: subs r2, r2, #0x20
! 114: bge Lmemcpy_floop32
! 115:
! 116: cmn r2, #0x10
! 117: ldmgeia r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */
! 118: stmgeia r0!, {r3, r4, r12, lr}
! 119: subge r2, r2, #0x10
! 120: ldmia sp!, {r4} /* return r4 */
! 121:
! 122: Lmemcpy_fl32:
! 123: adds r2, r2, #0x14
! 124:
! 125: /* blat 12 bytes at a time */
! 126: Lmemcpy_floop12:
! 127: ldmgeia r1!, {r3, r12, lr}
! 128: stmgeia r0!, {r3, r12, lr}
! 129: subges r2, r2, #0x0c
! 130: bge Lmemcpy_floop12
! 131:
! 132: Lmemcpy_fl12:
! 133: adds r2, r2, #8
! 134: blt Lmemcpy_fl4
! 135:
! 136: subs r2, r2, #4
! 137: ldrlt r3, [r1], #4
! 138: strlt r3, [r0], #4
! 139: ldmgeia r1!, {r3, r12}
! 140: stmgeia r0!, {r3, r12}
! 141: subge r2, r2, #4
! 142:
! 143: Lmemcpy_fl4:
! 144: /* less than 4 bytes to go */
! 145: adds r2, r2, #4
! 146: #ifdef __APCS_26_
! 147: ldmeqia sp!, {r0, pc}^ /* done */
! 148: #else
! 149: ldmeqia sp!, {r0, pc} /* done */
! 150: #endif
! 151: /* copy the crud byte at a time */
! 152: cmp r2, #2
! 153: ldrb r3, [r1], #1
! 154: strb r3, [r0], #1
! 155: ldrgeb r3, [r1], #1
! 156: strgeb r3, [r0], #1
! 157: ldrgtb r3, [r1], #1
! 158: strgtb r3, [r0], #1
! 159: #ifdef __APCS_26__
! 160: ldmia sp!, {r0, pc}^
! 161: #else
! 162: ldmia sp!, {r0, pc}
! 163: #endif
! 164:
! 165: /* erg - unaligned destination */
! 166: Lmemcpy_fdestul:
! 167: rsb r12, r12, #4
! 168: cmp r12, #2
! 169:
! 170: /* align destination with byte copies */
! 171: ldrb r3, [r1], #1
! 172: strb r3, [r0], #1
! 173: ldrgeb r3, [r1], #1
! 174: strgeb r3, [r0], #1
! 175: ldrgtb r3, [r1], #1
! 176: strgtb r3, [r0], #1
! 177: subs r2, r2, r12
! 178: blt Lmemcpy_fl4 /* less the 4 bytes */
! 179:
! 180: ands r12, r1, #3
! 181: beq Lmemcpy_ft8 /* we have an aligned source */
! 182:
! 183: /* erg - unaligned source */
! 184: /* This is where it gets nasty ... */
! 185: Lmemcpy_fsrcul:
! 186: bic r1, r1, #3
! 187: ldr lr, [r1], #4
! 188: cmp r12, #2
! 189: bgt Lmemcpy_fsrcul3
! 190: beq Lmemcpy_fsrcul2
! 191: cmp r2, #0x0c
! 192: blt Lmemcpy_fsrcul1loop4
! 193: sub r2, r2, #0x0c
! 194: stmdb sp!, {r4, r5}
! 195:
! 196: Lmemcpy_fsrcul1loop16:
! 197: mov r3, lr, lsr #8
! 198: ldmia r1!, {r4, r5, r12, lr}
! 199: orr r3, r3, r4, lsl #24
! 200: mov r4, r4, lsr #8
! 201: orr r4, r4, r5, lsl #24
! 202: mov r5, r5, lsr #8
! 203: orr r5, r5, r12, lsl #24
! 204: mov r12, r12, lsr #8
! 205: orr r12, r12, lr, lsl #24
! 206: stmia r0!, {r3-r5, r12}
! 207: subs r2, r2, #0x10
! 208: bge Lmemcpy_fsrcul1loop16
! 209: ldmia sp!, {r4, r5}
! 210: adds r2, r2, #0x0c
! 211: blt Lmemcpy_fsrcul1l4
! 212:
! 213: Lmemcpy_fsrcul1loop4:
! 214: mov r12, lr, lsr #8
! 215: ldr lr, [r1], #4
! 216: orr r12, r12, lr, lsl #24
! 217: str r12, [r0], #4
! 218: subs r2, r2, #4
! 219: bge Lmemcpy_fsrcul1loop4
! 220:
! 221: Lmemcpy_fsrcul1l4:
! 222: sub r1, r1, #3
! 223: b Lmemcpy_fl4
! 224:
! 225: Lmemcpy_fsrcul2:
! 226: cmp r2, #0x0c
! 227: blt Lmemcpy_fsrcul2loop4
! 228: sub r2, r2, #0x0c
! 229: stmdb sp!, {r4, r5}
! 230:
! 231: Lmemcpy_fsrcul2loop16:
! 232: mov r3, lr, lsr #16
! 233: ldmia r1!, {r4, r5, r12, lr}
! 234: orr r3, r3, r4, lsl #16
! 235: mov r4, r4, lsr #16
! 236: orr r4, r4, r5, lsl #16
! 237: mov r5, r5, lsr #16
! 238: orr r5, r5, r12, lsl #16
! 239: mov r12, r12, lsr #16
! 240: orr r12, r12, lr, lsl #16
! 241: stmia r0!, {r3-r5, r12}
! 242: subs r2, r2, #0x10
! 243: bge Lmemcpy_fsrcul2loop16
! 244: ldmia sp!, {r4, r5}
! 245: adds r2, r2, #0x0c
! 246: blt Lmemcpy_fsrcul2l4
! 247:
! 248: Lmemcpy_fsrcul2loop4:
! 249: mov r12, lr, lsr #16
! 250: ldr lr, [r1], #4
! 251: orr r12, r12, lr, lsl #16
! 252: str r12, [r0], #4
! 253: subs r2, r2, #4
! 254: bge Lmemcpy_fsrcul2loop4
! 255:
! 256: Lmemcpy_fsrcul2l4:
! 257: sub r1, r1, #2
! 258: b Lmemcpy_fl4
! 259:
! 260: Lmemcpy_fsrcul3:
! 261: cmp r2, #0x0c
! 262: blt Lmemcpy_fsrcul3loop4
! 263: sub r2, r2, #0x0c
! 264: stmdb sp!, {r4, r5}
! 265:
! 266: Lmemcpy_fsrcul3loop16:
! 267: mov r3, lr, lsr #24
! 268: ldmia r1!, {r4, r5, r12, lr}
! 269: orr r3, r3, r4, lsl #8
! 270: mov r4, r4, lsr #24
! 271: orr r4, r4, r5, lsl #8
! 272: mov r5, r5, lsr #24
! 273: orr r5, r5, r12, lsl #8
! 274: mov r12, r12, lsr #24
! 275: orr r12, r12, lr, lsl #8
! 276: stmia r0!, {r3-r5, r12}
! 277: subs r2, r2, #0x10
! 278: bge Lmemcpy_fsrcul3loop16
! 279: ldmia sp!, {r4, r5}
! 280: adds r2, r2, #0x0c
! 281: blt Lmemcpy_fsrcul3l4
! 282:
! 283: Lmemcpy_fsrcul3loop4:
! 284: mov r12, lr, lsr #24
! 285: ldr lr, [r1], #4
! 286: orr r12, r12, lr, lsl #8
! 287: str r12, [r0], #4
! 288: subs r2, r2, #4
! 289: bge Lmemcpy_fsrcul3loop4
! 290:
! 291: Lmemcpy_fsrcul3l4:
! 292: sub r1, r1, #1
! 293: b Lmemcpy_fl4
! 294:
! 295: Lmemcpy_backwards:
! 296: add r1, r1, r2
! 297: add r0, r0, r2
! 298: subs r2, r2, #4
! 299: blt Lmemcpy_bl4 /* less than 4 bytes */
! 300: ands r12, r0, #3
! 301: bne Lmemcpy_bdestul /* oh unaligned destination addr */
! 302: ands r12, r1, #3
! 303: bne Lmemcpy_bsrcul /* oh unaligned source addr */
! 304:
! 305: Lmemcpy_bt8:
! 306: /* We have aligned source and destination */
! 307: subs r2, r2, #8
! 308: blt Lmemcpy_bl12 /* less than 12 bytes (4 from above) */
! 309: stmdb sp!, {r4}
! 310: subs r2, r2, #0x14 /* less than 32 bytes (12 from above) */
! 311: blt Lmemcpy_bl32
! 312:
! 313: /* blat 32 bytes at a time */
! 314: /* XXX for really big copies perhaps we should use more registers */
! 315: Lmemcpy_bloop32:
! 316: ldmdb r1!, {r3, r4, r12, lr}
! 317: stmdb r0!, {r3, r4, r12, lr}
! 318: ldmdb r1!, {r3, r4, r12, lr}
! 319: stmdb r0!, {r3, r4, r12, lr}
! 320: subs r2, r2, #0x20
! 321: bge Lmemcpy_bloop32
! 322:
! 323: Lmemcpy_bl32:
! 324: cmn r2, #0x10
! 325: ldmgedb r1!, {r3, r4, r12, lr} /* blat a remaining 16 bytes */
! 326: stmgedb r0!, {r3, r4, r12, lr}
! 327: subge r2, r2, #0x10
! 328: adds r2, r2, #0x14
! 329: ldmgedb r1!, {r3, r12, lr} /* blat a remaining 12 bytes */
! 330: stmgedb r0!, {r3, r12, lr}
! 331: subge r2, r2, #0x0c
! 332: ldmia sp!, {r4}
! 333:
! 334: Lmemcpy_bl12:
! 335: adds r2, r2, #8
! 336: blt Lmemcpy_bl4
! 337: subs r2, r2, #4
! 338: ldrlt r3, [r1, #-4]!
! 339: strlt r3, [r0, #-4]!
! 340: ldmgedb r1!, {r3, r12}
! 341: stmgedb r0!, {r3, r12}
! 342: subge r2, r2, #4
! 343:
! 344: Lmemcpy_bl4:
! 345: /* less than 4 bytes to go */
! 346: adds r2, r2, #4
! 347: #ifdef __APCS_26__
! 348: ldmeqia sp!, {r0, pc}^
! 349: #else
! 350: ldmeqia sp!, {r0, pc}
! 351: #endif
! 352:
! 353: /* copy the crud byte at a time */
! 354: cmp r2, #2
! 355: ldrb r3, [r1, #-1]!
! 356: strb r3, [r0, #-1]!
! 357: ldrgeb r3, [r1, #-1]!
! 358: strgeb r3, [r0, #-1]!
! 359: ldrgtb r3, [r1, #-1]!
! 360: strgtb r3, [r0, #-1]!
! 361: #ifdef __APCS_26__
! 362: ldmia sp!, {r0, pc}^
! 363: #else
! 364: ldmia sp!, {r0, pc}
! 365: #endif
! 366:
! 367: /* erg - unaligned destination */
! 368: Lmemcpy_bdestul:
! 369: cmp r12, #2
! 370:
! 371: /* align destination with byte copies */
! 372: ldrb r3, [r1, #-1]!
! 373: strb r3, [r0, #-1]!
! 374: ldrgeb r3, [r1, #-1]!
! 375: strgeb r3, [r0, #-1]!
! 376: ldrgtb r3, [r1, #-1]!
! 377: strgtb r3, [r0, #-1]!
! 378: subs r2, r2, r12
! 379: blt Lmemcpy_bl4 /* less than 4 bytes to go */
! 380: ands r12, r1, #3
! 381: beq Lmemcpy_bt8 /* we have an aligned source */
! 382:
! 383: /* erg - unaligned source */
! 384: /* This is where it gets nasty ... */
! 385: Lmemcpy_bsrcul:
! 386: bic r1, r1, #3
! 387: ldr r3, [r1, #0]
! 388: cmp r12, #2
! 389: blt Lmemcpy_bsrcul1
! 390: beq Lmemcpy_bsrcul2
! 391: cmp r2, #0x0c
! 392: blt Lmemcpy_bsrcul3loop4
! 393: sub r2, r2, #0x0c
! 394: stmdb sp!, {r4, r5}
! 395:
! 396: Lmemcpy_bsrcul3loop16:
! 397: mov lr, r3, lsl #8
! 398: ldmdb r1!, {r3-r5, r12}
! 399: orr lr, lr, r12, lsr #24
! 400: mov r12, r12, lsl #8
! 401: orr r12, r12, r5, lsr #24
! 402: mov r5, r5, lsl #8
! 403: orr r5, r5, r4, lsr #24
! 404: mov r4, r4, lsl #8
! 405: orr r4, r4, r3, lsr #24
! 406: stmdb r0!, {r4, r5, r12, lr}
! 407: subs r2, r2, #0x10
! 408: bge Lmemcpy_bsrcul3loop16
! 409: ldmia sp!, {r4, r5}
! 410: adds r2, r2, #0x0c
! 411: blt Lmemcpy_bsrcul3l4
! 412:
! 413: Lmemcpy_bsrcul3loop4:
! 414: mov r12, r3, lsl #8
! 415: ldr r3, [r1, #-4]!
! 416: orr r12, r12, r3, lsr #24
! 417: str r12, [r0, #-4]!
! 418: subs r2, r2, #4
! 419: bge Lmemcpy_bsrcul3loop4
! 420:
! 421: Lmemcpy_bsrcul3l4:
! 422: add r1, r1, #3
! 423: b Lmemcpy_bl4
! 424:
! 425: Lmemcpy_bsrcul2:
! 426: cmp r2, #0x0c
! 427: blt Lmemcpy_bsrcul2loop4
! 428: sub r2, r2, #0x0c
! 429: stmdb sp!, {r4, r5}
! 430:
! 431: Lmemcpy_bsrcul2loop16:
! 432: mov lr, r3, lsl #16
! 433: ldmdb r1!, {r3-r5, r12}
! 434: orr lr, lr, r12, lsr #16
! 435: mov r12, r12, lsl #16
! 436: orr r12, r12, r5, lsr #16
! 437: mov r5, r5, lsl #16
! 438: orr r5, r5, r4, lsr #16
! 439: mov r4, r4, lsl #16
! 440: orr r4, r4, r3, lsr #16
! 441: stmdb r0!, {r4, r5, r12, lr}
! 442: subs r2, r2, #0x10
! 443: bge Lmemcpy_bsrcul2loop16
! 444: ldmia sp!, {r4, r5}
! 445: adds r2, r2, #0x0c
! 446: blt Lmemcpy_bsrcul2l4
! 447:
! 448: Lmemcpy_bsrcul2loop4:
! 449: mov r12, r3, lsl #16
! 450: ldr r3, [r1, #-4]!
! 451: orr r12, r12, r3, lsr #16
! 452: str r12, [r0, #-4]!
! 453: subs r2, r2, #4
! 454: bge Lmemcpy_bsrcul2loop4
! 455:
! 456: Lmemcpy_bsrcul2l4:
! 457: add r1, r1, #2
! 458: b Lmemcpy_bl4
! 459:
! 460: Lmemcpy_bsrcul1:
! 461: cmp r2, #0x0c
! 462: blt Lmemcpy_bsrcul1loop4
! 463: sub r2, r2, #0x0c
! 464: stmdb sp!, {r4, r5}
! 465:
! 466: Lmemcpy_bsrcul1loop32:
! 467: mov lr, r3, lsl #24
! 468: ldmdb r1!, {r3-r5, r12}
! 469: orr lr, lr, r12, lsr #8
! 470: mov r12, r12, lsl #24
! 471: orr r12, r12, r5, lsr #8
! 472: mov r5, r5, lsl #24
! 473: orr r5, r5, r4, lsr #8
! 474: mov r4, r4, lsl #24
! 475: orr r4, r4, r3, lsr #8
! 476: stmdb r0!, {r4, r5, r12, lr}
! 477: subs r2, r2, #0x10
! 478: bge Lmemcpy_bsrcul1loop32
! 479: ldmia sp!, {r4, r5}
! 480: adds r2, r2, #0x0c
! 481: blt Lmemcpy_bsrcul1l4
! 482:
! 483: Lmemcpy_bsrcul1loop4:
! 484: mov r12, r3, lsl #24
! 485: ldr r3, [r1, #-4]!
! 486: orr r12, r12, r3, lsr #8
! 487: str r12, [r0, #-4]!
! 488: subs r2, r2, #4
! 489: bge Lmemcpy_bsrcul1loop4
! 490:
! 491: Lmemcpy_bsrcul1l4:
! 492: add r1, r1, #1
! 493: b Lmemcpy_bl4
! 494:
CVSweb
![[BACK]](/icons/back.gif){kind=icon}
Return to memcpy.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / sys / lib / libkern / arch / arm