问题
My goal is to present 2D animated characters in the real environment using ARKit. The animated characters are part of a video at presented in the following snapshot from the video:
Displaying the video itself was achieved with no problem at all using the code:
func view(_ view: ARSKView, nodeFor anchor: ARAnchor) -> SKNode? {
guard let urlString = Bundle.main.path(forResource: "resourceName", ofType: "mp4") else { return nil }
let url = URL(fileURLWithPath: urlString)
let asset = AVAsset(url: url)
let item = AVPlayerItem(asset: asset)
let player = AVPlayer(playerItem: item)
let videoNode = SKVideoNode(avPlayer: player)
videoNode.size = CGSize(width: 200.0, height: 150.0)
videoNode.anchorPoint = CGPoint(x: 0.5, y: 0.0)
return videoNode
}
The result of this code is presented in the screen shot from the app below as expected:
But as you can see, the background of the characters isn't very nice, so I need to make it vanish, in order to create the illusion of the characters actually standing on the horizontal plane surface. I'm trying to achieve this by making a chroma-key effect to the video.
- For those who are not familiar with chroma-key, this is name of the "green screen effect" seen sometimes on TV to make a color transparent.
My approach to the chroma-key effect is to create a custom filter based on "CIColorCube" CIFilter, and then apply the filter to the video using AVVideoComposition.
First, is the code for creating the filter:
func RGBtoHSV(r : Float, g : Float, b : Float) -> (h : Float, s : Float, v : Float) {
var h : CGFloat = 0
var s : CGFloat = 0
var v : CGFloat = 0
let col = UIColor(red: CGFloat(r), green: CGFloat(g), blue: CGFloat(b), alpha: 1.0)
col.getHue(&h, saturation: &s, brightness: &v, alpha: nil)
return (Float(h), Float(s), Float(v))
}
func colorCubeFilterForChromaKey(hueAngle: Float) -> CIFilter {
let hueRange: Float = 20 // degrees size pie shape that we want to replace
let minHueAngle: Float = (hueAngle - hueRange/2.0) / 360
let maxHueAngle: Float = (hueAngle + hueRange/2.0) / 360
let size = 64
var cubeData = [Float](repeating: 0, count: size * size * size * 4)
var rgb: [Float] = [0, 0, 0]
var hsv: (h : Float, s : Float, v : Float)
var offset = 0
for z in 0 ..< size {
rgb[2] = Float(z) / Float(size) // blue value
for y in 0 ..< size {
rgb[1] = Float(y) / Float(size) // green value
for x in 0 ..< size {
rgb[0] = Float(x) / Float(size) // red value
hsv = RGBtoHSV(r: rgb[0], g: rgb[1], b: rgb[2])
// TODO: Check if hsv.s > 0.5 is really nesseccary
let alpha: Float = (hsv.h > minHueAngle && hsv.h < maxHueAngle && hsv.s > 0.5) ? 0 : 1.0
cubeData[offset] = rgb[0] * alpha
cubeData[offset + 1] = rgb[1] * alpha
cubeData[offset + 2] = rgb[2] * alpha
cubeData[offset + 3] = alpha
offset += 4
}
}
}
let b = cubeData.withUnsafeBufferPointer { Data(buffer: $0) }
let data = b as NSData
let colorCube = CIFilter(name: "CIColorCube", withInputParameters: [
"inputCubeDimension": size,
"inputCubeData": data
])
return colorCube!
}
And then the code for applying the filter to the video by modifying the function func view(_ view: ARSKView, nodeFor anchor: ARAnchor) -> SKNode? that I wrote earlier:
func view(_ view: ARSKView, nodeFor anchor: ARAnchor) -> SKNode? {
guard let urlString = Bundle.main.path(forResource: "resourceName", ofType: "mp4") else { return nil }
let url = URL(fileURLWithPath: urlString)
let asset = AVAsset(url: url)
let filter = colorCubeFilterForChromaKey(hueAngle: 38)
let composition = AVVideoComposition(asset: asset, applyingCIFiltersWithHandler: { request in
let source = request.sourceImage
filter.setValue(source, forKey: kCIInputImageKey)
let output = filter.outputImage
request.finish(with: output!, context: nil)
})
let item = AVPlayerItem(asset: asset)
item.videoComposition = composition
let player = AVPlayer(playerItem: item)
let videoNode = SKVideoNode(avPlayer: player)
videoNode.size = CGSize(width: 200.0, height: 150.0)
videoNode.anchorPoint = CGPoint(x: 0.5, y: 0.0)
return videoNode
}
The code is supposed to replace all pixels of each frame of the video to alpha = 0.0 if the pixel color match the hue range of the background.
But instead of getting transparent pixels I'm getting those pixels black as can be seen in the image below:
Now, even though this is not the wanted effect, it does not surprise me, as I knew that this is the way iOS displays videos with alpha channel.
But here is the real problem - When displaying a normal video in an AVPlayer, there is an option to add an AVPlayerLayer to the view, and to set pixelBufferAttributes to it, to let the player layer know we use a transparent pixel buffer, like so:
let playerLayer = AVPlayerLayer(player: player)
playerLayer.bounds = view.bounds
playerLayer.position = view.center
playerLayer.pixelBufferAttributes = [(kCVPixelBufferPixelFormatTypeKey as String): kCVPixelFormatType_32BGRA]
view.layer.addSublayer(playerLayer)
This code gives us a video with transparent background (GOOD!) but a fixed size and position (NOT GOOD...), as you can see in this screenshot:
I want to achieve the same effect, but on SKVideoNode, and not on AVPlayerLayer. However, I can't find any way to set pixelBufferAttributes to SKVideoNode, and setting a player layer does not achieve the desired effect of ARKit as it is fixed in position.
Is there any solution to my problem, or maybe is there another technique to achieve the same desired effect?
回答1:
The solution is quite simple!
All that needs to be done is to add the video as a child of a SKEffectNode and apply the filter to the SKEffectNode instead of the video itself (the AVVideoComposition is not necessary).
Here is the code I used:
func view(_ view: ARSKView, nodeFor anchor: ARAnchor) -> SKNode? {
// Create and configure a node for the anchor added to the view's session.
let bialikVideoNode = videoNodeWith(resourceName: "Tsina_05", ofType: "mp4")
bialikVideoNode.size = CGSize(width: kDizengofVideoWidth, height: kDizengofVideoHeight)
bialikVideoNode.anchorPoint = CGPoint(x: 0.5, y: 0.0)
// Make the video background transparent using an SKEffectNode, since chroma-key doesn't work on video
let effectNode = SKEffectNode()
effectNode.addChild(bialikVideoNode)
effectNode.filter = colorCubeFilterForChromaKey(hueAngle: 120)
return effectNode
}
And here is the result as needed:
回答2:
Thank you! Had the same problem + mixing [AR/Scene/Sprite]Kit. But I would recommend to use this algorithm instead. It gives a better result:
...
var r: [Float] = removeChromaKeyColor(r: rgb[0], g: rgb[1], b: rgb[2])
cubeData[offset] = r[0]
cubeData[offset + 1] = r[1]
cubeData[offset + 2] = r[2]
cubeData[offset + 3] = r[3]
offset += 4
...
func removeChromaKeyColor(r: Float, g: Float, b: Float) -> [Float] {
let threshold: Float = 0.1
let refColor: [Float] = [0, 1.0, 0, 1.0] // chroma key color
//http://www.shaderslab.com/demo-40---video-in-video-with-green-chromakey.html
let val = ceil(saturate(g - r - threshold)) * ceil(saturate(g - b - threshold))
var result = lerp(a: [r, g, b, 0.0], b: refColor, w: val)
result[3] = fabs(1.0 - result[3])
return result
}
func saturate(_ x: Float) -> Float {
return max(0, min(1, x));
}
func ceil(_ v: Float) -> Float {
return -floor(-v);
}
func lerp(a: [Float], b: [Float], w: Float) -> [Float] {
return [a[0]+w*(b[0]-a[0]), a[1]+w*(b[1]-a[1]), a[2]+w*(b[2]-a[2]), a[3]+w*(b[3]-a[3])];
}
来源：https://stackoverflow.com/questions/50139146/arkit-spritekit-set-pixelbufferattributes-to-skvideonode-or-make-transparent