first and last commit

This commit is contained in:
Chris Nutter committed 2023-07-08 13:03:17 -07:00
commit 1bf4efd552
2612 files changed
+636500

No files matched your search

+47
View File
@@ -0,0 +1,47 @@
/*
* Copyright (c) 2007 Apple Inc. All rights reserved.
*/
/*
* @OSF_COPYRIGHT@
*/
/*
* Mach Operating System
* Copyright (c) 1991,1990,1989 Carnegie Mellon University
* All Rights Reserved.
*
* Permission to use, copy, modify and distribute this software and its
* documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
*
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
*
* Carnegie Mellon requests users of this software to return to
*
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
*
* any improvements or extensions that they make and grant Carnegie Mellon
* the rights to redistribute these changes.
*/
#ifndef _ARM_ASM_H_
#define _ARM_ASM_H_
#ifdef __ASSEMBLER__
#define PUSH_FRAME \
stp fp, lr, [sp, #-16]! %% \
mov fp, sp %%
#define POP_FRAME \
mov sp, fp %% \
ldp fp, lr, [sp], #16 %%
#endif /* __ASSEMBLER__ */
#endif /* _ARM_ASM_H_ */
+294
View File
@@ -0,0 +1,294 @@
/*
* Copyright (c) 2012 Apple Computer, Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*
* This file implements the following functions for the arm64 architecture.
*
* void bcopy(const void * source,
* void * destination,
* size_t length);
*
* void *memmove(void * destination,
* const void * source,
* size_t n);
*
* void *memcpy(void * restrict destination,
* const void * restrict source,
* size_t n);
*
* All copy n successive bytes from source to destination. Memmove and memcpy
* return destination, whereas bcopy has no return value. Copying takes place
* as if it were through a temporary buffer -- after return destination
* contains exactly the bytes from source, even if the buffers overlap (this is
* not required of memcpy by the C standard; its behavior is undefined if the
* buffers overlap, but we are holding ourselves to the historical behavior of
* this function on MacOS).
*/
#include "asm.h"
.globl _bcopy
.globl _memcpy
.globl _memmove
/*****************************************************************************
* Macros *
*****************************************************************************/
#define kSmallCopy 64
/*****************************************************************************
* Entrypoints *
*****************************************************************************/
.text
.align 5
_bcopy:
// Translate bcopy into memcpy by swapping the first and second arguments.
mov x3, x0
mov x0, x1
mov x1, x3
.align 4
_memcpy:
_memmove:
// Our preference is to copy the data in ascending address order, but if the
// buffers overlap such that the beginning of the destination buffer aliases
// the end of the source buffer, we need to copy in descending address order
// instead to preserve the memmove semantics. We detect this case with the
// test:
//
// destination - source < length (unsigned compare)
//
// If the address of the source buffer is higher than the address of the
// destination buffer, this arithmetic can overflow, but the overflowed value
// can only be smaller than length if the buffers do not overlap, so we don't
// need to worry about false positives due to the overflow (they happen, but
// only in cases where copying in either order is correct).
PUSH_FRAME
sub x3, x0, x1
cmp x3, x2
b.cc L_reverse
mov x3, x0 // copy destination pointer
cmp x2, #(kSmallCopy)
b.cc L_forwardSmallCopy
/*****************************************************************************
* Forward large copy *
*****************************************************************************/
// Load the first 32 bytes from src, and compute the number of bytes to the
// first 32-byte aligned location in dst. Even though we are going to copy
// 32 bytes, only those preceeding that 32-byte location "count" towards
// reducing the length of the buffer or advancing the pointers. We will need
// to issue the first load from the advanced src pointer BEFORE the store to
// the unmodified dst pointer.
add x3, x3, #32
and x3, x3, #-32 // aligned dst
ldp x12,x13,[x1]
ldp x14,x15,[x1, #16]
sub x5, x3, x0 // bytes between original dst and aligned dst
add x1, x1, x5 // update src pointer
// At this point, data in the following registers is in flight:
//
// x0 original dst pointer
// x1 corresponding location in src buffer.
// x2 length from aligned location in dst to end of buffer. This is
// guaranteed to be >= (64 - 32).
// x3 aligned location in dst buffer.
// x12:x15 first 32 bytes of src buffer.
//
// We now load 32 bytes from x1, and store 32 bytes from x12:x15 to x3. The
// store *may* overlap the first 32 bytes of the load, so in order to get
// correct memmove semantics, the first 32 byte load must occur before the
// store.
//
// After loading these 32 bytes, we advance x1, and decrement the length by
// 64. If the remaining length of the buffer was less than 64, then we jump
// directly to the cleanup path.
ldp x8, x9, [x1]
ldp x10,x11,[x1, #16]
add x1, x1, #32
sub x2, x2, x5 // update length
stp x12,x13,[x0] // initial unaligned store
stp x14,x15,[x0, #16] // initial unaligned store
subs x2, x2, #64
b.ls L_forwardCleanup
L_forwardCopyLoop:
// Main copy loop:
//
// 1. store the 32 bytes loaded in the previous loop iteration
// 2. advance the destination pointer
// 3. load the next 32 bytes
// 4. advance the source pointer
// 5. subtract 32 from the length
//
// The loop is terminated when 32 or fewer bytes remain to be loaded. Those
// trailing 1-32 bytes will be copied in the loop cleanup.
stnp x8, x9, [x3]
stnp x10,x11,[x3, #16]
add x3, x3, #32
ldnp x8, x9, [x1]
ldnp x10,x11,[x1, #16]
add x1, x1, #32
subs x2, x2, #32
b.hi L_forwardCopyLoop
L_forwardCleanup:
// There are 32 bytes in x8-x11 that were loaded in the previous loop
// iteration, which need to be stored to [x3,x3+32). In addition, between
// 0 and 32 more bytes need to be copied from x1 to x3 + 32. The exact
// number of bytes to copy is x2 + 32. Instead of using smaller conditional
// copies, we simply copy 32 unaligned bytes from x1+x2 to 64+x3+x2.
// This copy may overlap with the first store, so the loads must come before
// the store of the data from the previous loop iteration.
add x1, x1, x2
ldp x12,x13,[x1]
ldp x14,x15,[x1, #16]
stp x8, x9, [x3]
stp x10,x11,[x3, #16]
add x3, x3, x2
stp x12,x13,[x3, #32]
stp x14,x15,[x3, #48]
POP_FRAME
ret
/*****************************************************************************
* forward small copy *
*****************************************************************************/
// Copy one quadword at a time until less than 8 bytes remain to be copied.
// At the point of entry to L_forwardSmallCopy, the "calling convention"
// is as follows:
//
// x0 pointer to first byte of destination
// x1 pointer to first byte of source
// x2 length of buffers
// x3 pointer to first byte of destination
0: ldr x6, [x1],#8
str x6, [x3],#8
L_forwardSmallCopy:
subs x2, x2, #8
b.cs 0b
adds x2, x2, #8
b.eq 2f
1: ldrb w6, [x1],#1
strb w6, [x3],#1
subs x2, x2, #1
b.ne 1b
2: POP_FRAME
ret
/*****************************************************************************
* Reverse copy engines *
*****************************************************************************/
// The reverse copy engines are identical in every way to the forward copy
// engines, except in that they do everything backwards. For this reason, they
// are somewhat more sparsely commented than the forward copy loops. I have
// tried to only comment things that might be somewhat surprising in how they
// differ from the forward implementation.
//
// The one important thing to note is that (almost without fail), x1 and x3
// will point to ONE BYTE BEYOND the "right-hand edge" of the active buffer
// throughout these copy loops. They are initially advanced to that position
// in the L_reverse jump island. Because of this, whereas the forward copy
// loops generally follow a "copy data, then advance pointers" scheme, in the
// reverse copy loops, we advance the pointers, then copy the data.
L_reverse:
// As a minor optimization, we early out if dst == src.
cbz x3, L_return
// advance both pointers to the ends of their respective buffers before
// jumping into the appropriate reverse copy loop.
add x4, x0, x2
add x1, x1, x2
cmp x2, #(kSmallCopy)
b.cc L_reverseSmallCopy
/*****************************************************************************
* Reverse large copy *
*****************************************************************************/
ldp x12,x13,[x1, #-16]
ldp x14,x15,[x1, #-32]
sub x3, x4, #1 // In the forward copy, we used dst+32 & -32
and x3, x3, #-32 // to find an aligned location in the dest
sub x5, x4, x3 // buffer. Here we use dst-1 & -32 instead,
sub x1, x1, x5 // because we are going backwards.
sub x2, x2, x5
ldp x8, x9, [x1, #-16]
ldp x10,x11,[x1, #-32]
stp x12,x13,[x4, #-16]
stp x14,x15,[x4, #-32]
sub x1, x1, #32
subs x2, x2, #64
b.ls L_reverseCleanup
L_reverseCopyLoop:
stnp x8, x9, [x3, #-16]
stnp x10,x11,[x3, #-32]
sub x3, x3, #32
ldnp x8, x9, [x1, #-16]
ldnp x10,x11,[x1, #-32]
sub x1, x1, #32
subs x2, x2, #32
b.hi L_reverseCopyLoop
L_reverseCleanup:
sub x1, x1, x2
ldp x12,x13,[x1, #-16]
ldp x14,x15,[x1, #-32]
stp x8, x9, [x3, #-16]
stp x10,x11,[x3, #-32]
stp x12,x13,[x0, #16] // In the forward copy, we need to compute the
stp x14,x15,[x0] // address of these stores, but here we already
POP_FRAME // have a pointer to the start of the buffer.
ret
/*****************************************************************************
* reverse small copy *
*****************************************************************************/
0: ldr x6, [x1,#-8]!
str x6, [x4,#-8]!
L_reverseSmallCopy:
subs x2, x2, #8
b.cs 0b
adds x2, x2, #8
b.eq 2f
1: ldrb w6, [x1,#-1]!
strb w6, [x4,#-1]!
subs x2, x2, #1
b.ne 1b
2: POP_FRAME
ret
L_return:
POP_FRAME
ret
+147
View File
@@ -0,0 +1,147 @@
/*
* Copyright (c) 2012 Apple Computer, Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*
* This file implements the following functions for the arm64 architecture:
*
* void bzero(void *buffer, size_t length);
* void __bzero(void *buffer, size_t length);
* void *memset(void *buffer, int value, size_t length);
*
* The first two zero-fill a buffer. The third fills the buffer with the low
* byte of its second argument.
*/
#include "asm.h"
.globl _bzero
.globl ___bzero
.globl _memset
/*****************************************************************************
* bzero entrypoint *
*****************************************************************************/
.text
.align 4
_bzero:
___bzero:
PUSH_FRAME
mov x2, x1
eor x1, x1, x1
mov x3, x0
cmp x2, #128
b.cc L_memsetSmall
/*****************************************************************************
* Large buffer zero engine *
*****************************************************************************/
L_bzeroLarge:
// Write the first 64 bytes of the buffer without regard to alignment, then
// advance x3 to point to a cacheline-aligned location within the buffer, and
// decrement the length accordingly.
stp x1, x1, [x0]
stp x1, x1, [x0, #16]
stp x1, x1, [x0, #32]
stp x1, x1, [x0, #48]
add x3, x0, #64
and x3, x3, #-64
add x2, x2, x0 // end of buffer
add x4, x3, #64 // end of first cacheline to zero
subs x2, x2, x4 // if the end of the buffer comes first, jump
b.ls 1f // directly to the cleanup pass.
0: dc zva, x3 // zero cacheline
add x3, x3, #64 // increment pointer
subs x2, x2, #64 // decrement length
b.hi 0b
1: add x3, x3, x2 // back up pointer to (end of buffer) - 64.
stp x1, x1, [x3] // and store 64 bytes to reach end of buffer.
stp x1, x1, [x3, #16]
stp x1, x1, [x3, #32]
stp x1, x1, [x3, #48]
POP_FRAME
ret
/*****************************************************************************
* memset entrypoint *
*****************************************************************************/
.align 4
_memset:
PUSH_FRAME
and x1, x1, #0xff
orr x3, xzr,#0x0101010101010101
mul x1, x1, x3
mov x3, x0
cmp x2, #64
b.cc L_memsetSmall
/*****************************************************************************
* Large buffer store engine *
*****************************************************************************/
L_memsetLarge:
// Write the first 64 bytes of the buffer without regard to alignment, then
// advance x3 to point to an aligned location within the buffer, and
// decrement the length accordingly.
stp x1, x1, [x0]
add x3, x0, #16
and x3, x3, #-16
add x2, x2, x0 // end of buffer
add x4, x3, #64 // end of first aligned 64-byte store
subs x2, x2, x4 // if the end of the buffer comes first, jump
b.ls 1f // directly to the cleanup store.
0: stnp x1, x1, [x3]
stnp x1, x1, [x3, #16]
stnp x1, x1, [x3, #32]
stnp x1, x1, [x3, #48]
add x3, x3, #64
subs x2, x2, #64
b.hi 0b
1: add x3, x3, x2 // back up pointer to (end of buffer) - 64.
stp x1, x1, [x3]
stp x1, x1, [x3, #16]
stp x1, x1, [x3, #32]
stp x1, x1, [x3, #48]
POP_FRAME
ret
/*****************************************************************************
* Small buffer store engine *
*****************************************************************************/
0: str x1, [x3],#8
L_memsetSmall:
subs x2, x2, #8
b.cs 0b
adds x2, x2, #8
b.eq 2f
1: strb w1, [x3],#1
subs x2, x2, #1
b.ne 1b
2: POP_FRAME
ret
+24
View File
@@ -0,0 +1,24 @@
/*
* Copyright (C) 2012 Apple Computer, Inc. All rights reserved.
*
* This document is the property of Apple Computer, Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Computer, Inc.
*/
/* void memcpy_16(uint8_t *dest, const void * src, size_t count); */
.text
.align 2
.globl _memcpy_16
_memcpy_16:
mov x3, x0
mov x4, x1
L__memcpy_16_loop:
ldp x5, x6, [x4], #16
stp x5, x6, [x3], #16
subs x2, x2, #16
b.ne L__memcpy_16_loop
ret
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright (C) 2012 Apple Computer, Inc. All rights reserved.
*
* This document is the property of Apple Computer, Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Computer, Inc.
*/
/* void memset_16(uint8_t *ptr, uint8_t c, size_t count); */
.text
.align 2
.globl _memset_16
_memset_16:
mov x3, x0
cbz x1, L__memset_16_loop
orr x1, x1, x1, lsl #8
orr x1, x1, x1, lsl #16
orr x1, x1, x1, lsl #32
L__memset_16_loop:
stp x1, x1, [x3], #16
subs x2, x2, #16
b.gt L__memset_16_loop
ret
/* void bzero_64(uint8_t *ptr, size_t count); */
.text
.align 2
.globl _bzero_64
_bzero_64:
mov x2, x0
L__bzero_64_loop:
dc zva, x2
add x2, x2, #64
subs x1, x1, #64
b.gt L__bzero_64_loop
ret
+21
View File
@@ -0,0 +1,21 @@
# Copyright (C) 2011-2012 Apple Inc. All rights reserved.
#
# This document is the property of Apple Inc.
# It is considered confidential and proprietary.
#
# This document may not be reproduced or transmitted in any form,
# in whole or in part, without the express written permission of
# Apple Inc.
#
LOCAL_DIR := $(GET_LOCAL_DIR)
LIBC_OBJS += \
$(LOCAL_DIR)/bcopy.o \
$(LOCAL_DIR)/bzero.o
ARCH_STROPS := \
bcopy \
bzero \
memset \
memmove \
memcpy