I am using AVFoundation to merge two videos into one. The result of my attempt is a single video with a length equal to the sum of all the clips, and displaying a black screen.
Here is my code:
public void mergeclips()
{
AVMutableComposition mixComposition = new AVMutableComposition();
CMTime previous_asset_duration = CMTime.Zero;
CMTime AllAssetDurations = CMTime.Zero;
AVMutableVideoCompositionLayerInstruction[] Instruction_Array = new AVMutableVideoCompositionLayerInstruction[Clips.Count];
foreach (string clip in Clips)
{
#region HoldVideoTrack
AVAsset asset = AVAsset.FromUrl(NSUrl.FromFilename(clip));
AVMutableCompositionTrack Track = mixComposition.AddMutableTrack(AVMediaType.Video, 0);
CMTimeRange range = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = asset.Duration
};
AVAssetTrack track = asset.TracksWithMediaType(AVMediaType.Video)[0];
Track.InsertTimeRange(range, track, previous_asset_duration, out NSError error);
#endregion
#region Instructions
// 7
var Instruction = AVMutableVideoCompositionLayerInstruction.FromAssetTrack(Track);
Instruction.SetOpacity(0, asset.Duration);
// 8
Instruction_Array[Clips.IndexOf(clip)] = Instruction;
#endregion
previous_asset_duration = asset.Duration;
AllAssetDurations = asset.Duration;
}
// 6
var mainInstruction = new List<AVMutableVideoCompositionInstruction>();
CMTimeRange rangeIns = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = AllAssetDurations
};
mainInstruction[0].TimeRange = rangeIns;
mainInstruction[0].LayerInstructions = Instruction_Array;
var mainComposition = new AVMutableVideoComposition();
mainComposition.Instructions = mainInstruction.ToArray();
mainComposition.FrameDuration = new CMTime(1, 30);
mainComposition.RenderSize = new CoreGraphics.CGSize(UIScreen.MainScreen.Bounds.Width, UIScreen.MainScreen.Bounds.Height);
//... export video ...
AVAssetExportSession exportSession = new AVAssetExportSession(mixComposition, AVAssetExportSessionPreset.MediumQuality)
{
OutputUrl = NSUrl.FromFilename(Path.Combine(Path.GetTempPath(), "temporaryClip/Whole.mov")),
OutputFileType = AVFileType.QuickTimeMovie,
ShouldOptimizeForNetworkUse = true,
//APP crashes here
VideoComposition = mainComposition
};
exportSession.ExportAsynchronously(_OnExportDone);
}
private static void _OnExportDone()
{
var library = new ALAssetsLibrary();
library.WriteVideoToSavedPhotosAlbum(NSUrl.FromFilename(Path.Combine(Path.GetTempPath(), "temporaryClip/Whole.mov")), (path, e2) =>
{
if (e2 != null)
{
new UIAlertView("Error", e2.ToString(), null, "OK", null).Show();
}
else
{
}
});
}
EDIT:
I added more code, specifically, I added "ShouldOptimizeForNetworkUse" and VideoCompositions to the AVAssetExportSession.
I am using List instead of AVMutableVideoCompositionInstruction because AVMutableVideoComposition.Instructions requires a class of type AVVideoCompositionInstructions[].
With the previous code the App crashes at the following line "VideoComposition = mainComposition"
EDIT: After including transformations for the instructions and making the corrections that Shawn pointed out, I can merge 2 or more videos and save the common video to a file. Unfortunately, the root problem remains, the final video displays only the backgroundColor of AVMutableVideoCompositionInstruction, not all the clips as we would expect. The audio of these videos is also ignored, I don't know if this has to be added apart or not, but knowing it might also be helpful.
Here is my code:
public void mergeclips()
{
AVMutableComposition mixComposition = new AVMutableComposition();
AVMutableVideoCompositionLayerInstruction[] Instruction_Array = new AVMutableVideoCompositionLayerInstruction[Clips.Count];
foreach (string clip in Clips)
{
#region HoldVideoTrack
AVAsset asset = AVAsset.FromUrl(NSUrl.FromFilename(clip));
AVMutableCompositionTrack Track = mixComposition.AddMutableTrack(AVMediaType.Video, 0);
CMTimeRange range = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = asset.Duration
};
AVAssetTrack track = asset.TracksWithMediaType(AVMediaType.Video)[0];
Track.InsertTimeRange(range, track, mixComposition.Duration, out NSError error);
#endregion
#region Instructions
Instruction_Array[Clips.IndexOf(clip)] = SetInstruction(asset, mixComposition.Duration, Track);
#endregion
}
// 6
var mainInstruction = new AVMutableVideoCompositionInstruction();
CMTimeRange rangeIns = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = mixComposition.Duration
};
mainInstruction.BackgroundColor = UIColor.FromRGBA(1f, 1f, 1f, 1.000f).CGColor;
mainInstruction.TimeRange = rangeIns;
mainInstruction.LayerInstructions = Instruction_Array;
var mainComposition = new AVMutableVideoComposition()
{
Instructions = new AVVideoCompositionInstruction[1] { mainInstruction },
FrameDuration = new CMTime(1, 30),
RenderSize = new CoreGraphics.CGSize(UIScreen.MainScreen.Bounds.Width, UIScreen.MainScreen.Bounds.Height)
};
//... export video ...
AVAssetExportSession exportSession = new AVAssetExportSession(mixComposition, AVAssetExportSessionPreset.MediumQuality)
{
OutputUrl = NSUrl.FromFilename(Path.Combine(Path.GetTempPath(), "temporaryClip/Whole.mov")),
OutputFileType = AVFileType.QuickTimeMovie,
ShouldOptimizeForNetworkUse = true,
VideoComposition = mainComposition
};
exportSession.ExportAsynchronously(_OnExportDone);
}
private AVMutableVideoCompositionLayerInstruction SetInstruction(AVAsset asset, CMTime currentTime, AVMutableCompositionTrack assetTrack)
{
var instruction = AVMutableVideoCompositionLayerInstruction.FromAssetTrack(assetTrack);
var transform = assetTrack.PreferredTransform;
var transformSize = assetTrack.NaturalSize; //for export session
var newAssetSize = new CoreGraphics.CGSize(transformSize.Width, transformSize.Height); // for export session
if (newAssetSize.Width > newAssetSize.Height)//portrait
{
//Starting here, all newassetsize have its height and width inverted, height should be width and vice versa
var scaleRatio = UIScreen.MainScreen.Bounds.Height / newAssetSize.Width;
var _coreGraphic = new CoreGraphics.CGAffineTransform(0, 0, 0, 0, 0, 0);
_coreGraphic.Scale(scaleRatio, scaleRatio);
var tx = UIScreen.MainScreen.Bounds.Width / 2 - newAssetSize.Height * scaleRatio / 2;
var ty = UIScreen.MainScreen.Bounds.Height / 2 - newAssetSize.Width * scaleRatio / 2;
_coreGraphic.Translate(tx, ty);
instruction.SetTransform(_coreGraphic, currentTime);
}
var endTime = CMTime.Add(currentTime, asset.Duration);
instruction.SetOpacity(0, endTime);
return instruction;
}
EDIT: Several mistakes in the code were corrected thanks to Shawn's help. The problem remains (the resulting video has no image)
Here is my code:
public void mergeclips()
{
//microphone
AVCaptureDevice microphone = AVCaptureDevice.DefaultDeviceWithMediaType(AVMediaType.Audio);
AVMutableComposition mixComposition = new AVMutableComposition();
AVMutableVideoCompositionLayerInstruction[] Instruction_Array = new AVMutableVideoCompositionLayerInstruction[Clips.Count];
foreach (string clip in Clips)
{
#region HoldVideoTrack
AVAsset asset = AVAsset.FromUrl(NSUrl.FromFilename(clip));
CMTimeRange range = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = asset.Duration
};
AVMutableCompositionTrack videoTrack = mixComposition.AddMutableTrack(AVMediaType.Video, 0);
AVAssetTrack assetVideoTrack = asset.TracksWithMediaType(AVMediaType.Video)[0];
videoTrack.InsertTimeRange(range, assetVideoTrack, mixComposition.Duration, out NSError error);
if (microphone != null)
{
AVMutableCompositionTrack audioTrack = mixComposition.AddMutableTrack(AVMediaType.Audio, 0);
AVAssetTrack assetAudioTrack = asset.TracksWithMediaType(AVMediaType.Audio)[0];
audioTrack.InsertTimeRange(range, assetAudioTrack, mixComposition.Duration, out NSError error2);
}
#endregion
#region Instructions
Instruction_Array[Clips.IndexOf(clip)] = SetInstruction(asset, mixComposition.Duration, videoTrack);
#endregion
}
// 6
var mainInstruction = new AVMutableVideoCompositionInstruction();
CMTimeRange rangeIns = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = mixComposition.Duration
};
mainInstruction.BackgroundColor = UIColor.FromRGBA(1f, 1f, 1f, 1.000f).CGColor;
mainInstruction.TimeRange = rangeIns;
mainInstruction.LayerInstructions = Instruction_Array;
var mainComposition = new AVMutableVideoComposition()
{
Instructions = new AVVideoCompositionInstruction[1] { mainInstruction },
FrameDuration = new CMTime(1, 30),
RenderSize = new CoreGraphics.CGSize(UIScreen.MainScreen.Bounds.Width, UIScreen.MainScreen.Bounds.Height)
};
//... export video ...
AVAssetExportSession exportSession = new AVAssetExportSession(mixComposition, AVAssetExportSessionPreset.MediumQuality)
{
OutputUrl = NSUrl.FromFilename(Path.Combine(Path.GetTempPath(), "temporaryClip/Whole.mov")),
OutputFileType = AVFileType.QuickTimeMovie,
ShouldOptimizeForNetworkUse = true,
VideoComposition = mainComposition
};
exportSession.ExportAsynchronously(_OnExportDone);
}
private AVMutableVideoCompositionLayerInstruction SetInstruction(AVAsset asset, CMTime currentTime, AVMutableCompositionTrack mixComposition_video_Track)
{
//The following code triggers when a device has no camera or no microphone (for instance an emulator)
var instruction = AVMutableVideoCompositionLayerInstruction.FromAssetTrack(mixComposition_video_Track);
//Get the individual AVAsset's track to use for transform
AVAssetTrack assetTrack = asset.TracksWithMediaType(AVMediaType.Video)[0];
//Set transform the the preferredTransform of the AVAssetTrack, not the AVMutableCompositionTrack
CGAffineTransform transform = assetTrack.PreferredTransform;
//Set the transformSize to be the asset natural size AFTER applying preferredTransform.
CGSize transformSize = transform.TransformSize(assetTrack.NaturalSize);
//Handle any negative values resulted from applying transform by using the absolute value
CGSize newAssetSize = new CoreGraphics.CGSize(Math.Abs(transformSize.Width), Math.Abs(transformSize.Height));
//change back to less than
if (newAssetSize.Width < newAssetSize.Height)//portrait
{
/*newAssetSize should no longer be inverted since preferredTransform handles this. Remember that the asset was never
* actually transformed yet. newAssetSize just represents the size the video is going to be after you call
* instruction.setTransform(transform). Since transform is the first transform in concatenation, this is the size that
* the scale and translate transforms will be using, which is why we needed to reference newAssetSize after applying
* transform. Also you should concatenate in this order: transform -> scale -> translate, otherwise you won't get
* desired results*/
nfloat scaleRatio = UIScreen.MainScreen.Bounds.Height / newAssetSize.Height;
//Apply scale to transform. Transform is never actually applied unless you do this.
transform.Scale(scaleRatio, scaleRatio);
nfloat tx = UIScreen.MainScreen.Bounds.Width / 2 - newAssetSize.Width * scaleRatio / 2;
nfloat ty = UIScreen.MainScreen.Bounds.Height / 2 - newAssetSize.Height * scaleRatio / 2;
transform.Translate(tx, ty);
instruction.SetTransform(transform, currentTime);
}
var endTime = CMTime.Add(currentTime, asset.Duration);
instruction.SetOpacity(0, endTime);
return instruction;
}
You are inserting every time range at CMTime.zero instead of at the end of the previous asset. Also, are you using a videoComposition when you export?
UPDATE: A long time ago I was doing video playback within the app so I don't actually export, but when I first started I exported first then passed the exported video as an AVAsset into AVPlayer. I changed a lot since then so I don't export the video just to playback within the app because it was inefficient and a waste of time, but my code works perfectly in terms of merging assets together. I had it working when I was exporting, but I also changed my merge function a lot since then so no guarantees this will work with an export session.
func mergeVideos(mixComposition: Binding<AVMutableComposition>, videoComposition: Binding<AVMutableVideoComposition>, mainInstruction: Binding<AVMutableVideoCompositionInstruction>) -> AVPlayerItem {
guard let documentDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else {
return AVPlayerItem(asset: mixComposition.wrappedValue)
}
//Remove all existing videos, tracks and instructions
self.assets.removeAll()
for track in mixComposition.wrappedValue.tracks {
mixComposition.wrappedValue.removeTrack(track)
}
//Add all videos to asset array
for video in videos {
let url = documentDirectory.appendingPathComponent(video.videoURL)
let asset = AVURLAsset(url: url, options: [AVURLAssetPreferPreciseDurationAndTimingKey : true])
self.assets.append(asset)
}
//add instructions and assets to mixComposition
assets.forEach { asset in
self.addTrackToComposition(asset: asset, mixComposition: mixComposition, videoComposition: videoComposition, mainInstruction: mainInstruction)
}//forEach
//create playerITem with videoComposition
videoComposition.wrappedValue.instructions = [mainInstruction.wrappedValue]
videoComposition.wrappedValue.frameDuration = CMTimeMake(value: 1, timescale: 30)
videoComposition.wrappedValue.renderSize = renderSize
let item = AVPlayerItem(asset: mixComposition.wrappedValue)
item.seekingWaitsForVideoCompositionRendering = true
item.videoComposition = videoComposition.wrappedValue
return item
}//mergeVideo
func addTrackToComposition(asset: AVAsset, mixComposition: Binding<AVMutableComposition>, videoComposition: Binding<AVMutableVideoComposition>, mainInstruction: Binding<AVMutableVideoCompositionInstruction>) {
let currentTime = mixComposition.wrappedValue.duration
guard let assetVideoTrack = mixComposition.wrappedValue.addMutableTrack(withMediaType: .video, preferredTrackID: Int32(kCMPersistentTrackID_Invalid)) else {return}
guard let assetAudioTrack = mixComposition.wrappedValue.addMutableTrack(withMediaType: .audio, preferredTrackID: Int32(kCMPersistentTrackID_Invalid)) else {return}
do {
let timeRange = CMTimeRangeMake(start: .zero, duration: asset.duration)
// Insert video to Mutable Composition at right time.
try assetVideoTrack.insertTimeRange(timeRange, of: asset.tracks(withMediaType: .video)[0], at: currentTime)
try assetAudioTrack.insertTimeRange(timeRange, of: asset.tracks(withMediaType: .audio)[0], at: currentTime)
let videoInstruction = videoCompositionInstruction(track: assetVideoTrack, asset: asset, currentTime: currentTime)
mainInstruction.wrappedValue.layerInstructions.append(videoInstruction)
mainInstruction.wrappedValue.timeRange = CMTimeRange(start: .zero, duration: mixComposition.wrappedValue.duration)
} catch let error {
print(error.localizedDescription)
}
}//addTrackToComposition
func videoCompositionInstruction(track: AVCompositionTrack, asset: AVAsset, currentTime: CMTime) -> AVMutableVideoCompositionLayerInstruction {
let instruction = AVMutableVideoCompositionLayerInstruction(assetTrack: track)
guard let assetTrack = asset.tracks(withMediaType: .video).first else { return instruction }
let transform = assetTrack.preferredTransform
let transformSize = assetTrack.naturalSize.applying(transform) //for export session
let newAssetSize = CGSize(width: abs(transformSize.width), height: abs(transformSize.height)) // for export session
if newAssetSize.width < newAssetSize.height { //portrait
let scaleRatio = renderSize.height / newAssetSize.height
let scale = CGAffineTransform(scaleX: scaleRatio, y: scaleRatio)
let tx = renderSize.width / 2 - newAssetSize.width * scaleRatio / 2
let ty = renderSize.height / 2 - newAssetSize.height * scaleRatio / 2
let translate = CGAffineTransform(translationX: tx, y: ty)
let concatenation = transform.concatenating(scale).concatenating(translate)
instruction.setTransform(concatenation, at: currentTime)
} else if newAssetSize.width > newAssetSize.height { //landscape
let scaleRatio = renderSize.width / newAssetSize.width
let scale = CGAffineTransform(scaleX: scaleRatio, y: scaleRatio)
let tx = renderSize.width / 2 - newAssetSize.width * scaleRatio / 2
let ty = renderSize.height / 2 - newAssetSize.height * scaleRatio / 2
let translate = CGAffineTransform(translationX: tx, y: ty)
let concatenation = transform.concatenating(scale).concatenating(translate)
instruction.setTransform(concatenation, at: currentTime)
} else if newAssetSize.width == newAssetSize.height {
//if landscape, fill height first, if portrait fill width first, if square doesnt matter just scale either width or height
if renderSize.width > renderSize.height { //landscape
let scaleRatio = renderSize.height / newAssetSize.height
let scale = CGAffineTransform(scaleX: scaleRatio, y: scaleRatio)
let tx = renderSize.width / 2 - newAssetSize.width * scaleRatio / 2
let ty = renderSize.height / 2 - newAssetSize.height * scaleRatio / 2
let translate = CGAffineTransform(translationX: tx, y: ty)
let concatenation = transform.concatenating(scale).concatenating(translate)
instruction.setTransform(concatenation, at: currentTime)
} else { //portrait and square
let scaleRatio = renderSize.width / newAssetSize.width
let scale = CGAffineTransform(scaleX: scaleRatio, y: scaleRatio)
let tx = renderSize.width / 2 - newAssetSize.width * scaleRatio / 2
let ty = renderSize.height / 2 - newAssetSize.height * scaleRatio / 2
let translate = CGAffineTransform(translationX: tx, y: ty)
let concatenation = transform.concatenating(scale).concatenating(translate)
instruction.setTransform(concatenation, at: currentTime)
}
}
let endTime = CMTimeAdd(currentTime, asset.duration)
instruction.setOpacity(0, at: endTime)
return instruction
}//videoCompositionInstruction
I'll briefly explain what I am doing here.
You don't need to pass in Bindings for the AVMutableComposition, AVMutableVideoComposition or the AVMutableVideoCompositionInstructions. I only do that for certain functionalities in my app. You can instantiate all of these within in the function before you do anything else.
I have an array in the class that holds all my assets, so thats what self.assets is. "videos" is referencing a Realm Model I use to store the last path component of the videos the user picked from their photo library. You probably don't need to remove all existing videos, tracks, and instructions since you are not passing in references to the compositions and instructions. I do so since I make changes to these objects throughout my app. No need for you to use any wrappedValues either since that is just for the Binding.
Once I populate my assets array, I iterate through it calling addTrackToComposition and passing in each asset. This function adds an audio AND video track to the mixComposition for each asset. Then inside a do-catch block, it tries to insert the assets audio and video track into the empty mutableTracks you just created for the mixComposition. So the mixComposition is going to have 2 tracks for every asset (one audio and one video). I do this so that I can have more control over my instructions and apply different transforms to each asset rather than the entire mixComposition as a whole. You can also just create the empty mutableTracks for the mixComposition outside of the for loop and insert the asset's track into that one track (actually two tracks - audio/video). I know this sounds confusing by try to break it down slowly. It is important to note that in my do-catch block, the timeRange I pass is the assets time range, but I insert it at the end of the mixComposition (currentTime = mixComposition.duration). This is why timeRange starts at kCMTimeZero (.zero), but I pass in currentTime for the at: parameter.
Then I use a function that creates the layer instruction for each asset. This scales and positions each asset so that it is displayed properly in my custom video player. It also sets the opacity to 0 at the end of asset. Here my renderSize is declared in my Realm Model and is a CGSize(width: 1280, height: 720). Now, I am not sure if these transforms will work for your use case, but I know you will definitely need transforms otherwise your assets will export with the wrong orientation and/or size/position. At the very least you need to set the assets preferredTrackTransform. It is important to use the AVAssetTrack's preferredTransform and not the AVCompositionTrack's preferredTransform. This handles orientation for you, but will not handle scale and position. Play around with it. This code took me 1-2 weeks to figure out how to make work for my use case.
Then I append the layer instruction to mainInstruction and set mainInstructions timeRange to equal mixCompositions timeRange. I don't know why I set the timeRange every iteration of the for loop and I could definitely just do it after all instructions and tracks are added so it only happens once rather than every iteration.
Lastly, I set videoCompositions instruction to be an array of just mainInstruction and I set the frame rate and renderSize. Hopefully all of this works for you when you pass it into an export session.
Looking at the way you tried to implement it, I would say you don't need an array for layerInstructions. Just create an AVMutableVideoCompositionInstruction object (mainInstruction) and append the layer instructions to that.
Also there is a problem with you using previous asset duration. You need to pass in mixCompositions duration when you insert the new asset's time range. What you are doing is inserting at just the previous assets duration so you are ending up with a bunch of overlapping assets. You want to insert it after all previous assets duration combined, which would mixCompositions current duration.
Also mainInstruction should not be a List either. It should just be an AVMutableVideoCompositionInstruction(). AVMutableVideoCompositionInstruction has a layerInstructions property that is an array of layerInstructions. You can append directly to this. There should not be more than one mainInstruction. There should only be multiple layerInstructions.
Be patient with this. It took me a very long time to figure out myself, coming from no AVFoundation experience. I honestly still don't know enough to be sure of what's wrong with your current code, but all I know is that this works for me. Hopefully, it works for you too. I've probably changed this function 20 times since I started this app a couple months ago.
UPDATE: So you are on the right path, but there are still a few problems that may be the cause of your issue.
1.) I forgot to mention this last time, but when I was faced with the same problem, multiple people told me that I HAD to handle the audio track separately. Apparently even the video won't work without doing this. I never actually tested to see if this was true, but it's worth a shot. You can refer to my code again to see how I handled the audio track. It is essentially the same thing as the video track but you don't apply any instructions to it.
2.) In your instructions function there are few problems. Your instruction property is correct, but your transform, transformSize, and newAssetSize are not correct. Currently you set transform to assetTrack.preferredTransform. This is actually the mixComposition's transform, but what you want to use is the original AVAsset's preferredTransform.
After you initialize your instruction using assetTrack (mixComposition's track), you need to declare a new property to get the AVAsset's track. Refer back to my code. I actually use the name "assetTrack" to do this so don't be confused with our variable names. Your "assetTrack" is my "track" which I passed in as a parameter. My "assetTrack" is what you need to add, but obviously you can use whatever name you want.
So videos are a little strange when recorded on our devices. A video recorded in portrait orientation is actually landscape. Each asset, however, comes with data that informs the device how it should be displayed (i.e. rotate video so it displays the same way it was recorded). That is what preferredTransform is. It will transform the asset to display in the correct orientation. This is why you need to make sure you are using each individual asset's preferredTransform and not the mixComposition's preferredTransform that you used in your code. The mixComposition's preferredTransform will just be an identity matrix which effectively doesn't do anything at all. This is why your asset's natural size is "inverted". It is not inverted, that is just the way apple stores all videos and pictures. The meta data handles the correct orientation which is in preferredTransform and that will result in the "correct" width and height.
So now that you have the correct transform stored in your transform property, your transformSize property needs to reflect this, however you forget to add "applying(transform)" to the size. This is important. The transformSize you currently have is just the naturalSize, whereas, you want the size after applying the transform to the asset (so that width and height actually reflect the correct orientation of the video).
So now newAssetSize is meant to handle any negative values that are resulted from transformSize. So when you create newAssetSize, you need to make sure you are using the Absolute value of transformSize.width and transformSize.height. That is why I had it as "abs(transformSize.width)" in my code. This is also crucial.
3.) You never applied the preferredTransform to the video, and you instead apply scale transform to a matrix of all 0, which is never going to work. At very least you need to concatenate scale and translate to the identity matrix, although you should really be concatenating them to transform instead. If you don't change this part your video will never display no matter what you do. Any transforms you concatenate with on a zero matrix will have no effect and you will still result in a 0 matrix which means your video will not display at all.
Try to make these changes, especially the changes in the instruction function. I believe you will also have to redo your transform logic after you change those properties as it looks like you tried to compensate for the fact that width and height were inverted.
Your code should look something like this (keep in mind that I am not familiar with c# at all):
public void mergeclips()
{
AVMutableComposition mixComposition = new AVMutableComposition();
AVMutableVideoCompositionLayerInstruction[] Instruction_Array = new AVMutableVideoCompositionLayerInstruction[Clips.Count];
foreach (string clip in Clips)
{
#region HoldVideoTrack
AVAsset asset = AVAsset.FromUrl(NSUrl.FromFilename(clip));
AVMutableCompositionTrack videoTrack = mixComposition.AddMutableTrack(AVMediaType.Video, 0);
AVMutableCompositionTrack audioTrack = mixComposition.AddMutableTrack(AVMediaType.Audio, 0);
CMTimeRange range = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = asset.Duration
};
AVAssetTrack assetVideoTrack = asset.TracksWithMediaType(AVMediaType.Video)[0];
videoTrack.InsertTimeRange(range, assetVideoTrack, mixComposition.Duration, out NSError error);
AVAssetTrack assetAudioTrack = asset.TracksWithMediaType(AVMediaType.Audio)[0];
audioTrack.InsertTimeRange(range, assetAudioTrack, mixComposition.Duration, out NSError error);
#endregion
#region Instructions
Instruction_Array[Clips.IndexOf(clip)] = SetInstruction(asset, mixComposition.Duration, videoTrack);
#endregion
}
// 6
var mainInstruction = new AVMutableVideoCompositionInstruction();
CMTimeRange rangeIns = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = mixComposition.Duration
};
mainInstruction.BackgroundColor = UIColor.FromRGBA(1f, 1f, 1f, 1.000f).CGColor;
mainInstruction.TimeRange = rangeIns;
mainInstruction.LayerInstructions = Instruction_Array;
var mainComposition = new AVMutableVideoComposition()
{
Instructions = new AVVideoCompositionInstruction[1] { mainInstruction },
FrameDuration = new CMTime(1, 30),
RenderSize = new CoreGraphics.CGSize(UIScreen.MainScreen.Bounds.Width, UIScreen.MainScreen.Bounds.Height)
};
//... export video ...
AVAssetExportSession exportSession = new AVAssetExportSession(mixComposition, AVAssetExportSessionPreset.MediumQuality)
{
OutputUrl = NSUrl.FromFilename(Path.Combine(Path.GetTempPath(), "temporaryClip/Whole.mov")),
OutputFileType = AVFileType.QuickTimeMovie,
ShouldOptimizeForNetworkUse = true,
VideoComposition = mainComposition
};
exportSession.ExportAsynchronously(_OnExportDone);
}
private AVMutableVideoCompositionLayerInstruction SetInstruction(AVAsset asset, CMTime currentTime, AVMutableCompositionTrack assetTrack)
{
var instruction = AVMutableVideoCompositionLayerInstruction.FromAssetTrack(assetTrack);
//Get the individual AVAsset's track to use for transform
AVAssetTrack avAssetTrack = asset.TracksWithMediaType(AVMediaType.Video)[0];
//Set transform the the preferredTransform of the AVAssetTrack, not the AVMutableCompositionTrack
var transform = avAssetTrack.PreferredTransform;
//Set the transformSize to be the asset natural size AFTER applying preferredTransform.
var transformSize = avAssetTrack.NaturalSize.applying(transform);
//Handle any negative values resulted from applying transform by using the absolute value
var newAssetSize = new CoreGraphics.CGSize(Abs(transformSize.Width), Abs(transformSize.Height)); // for export session
//change back to less than
if (newAssetSize.Width < newAssetSize.Height)//portrait
{
//newAssetSize should no longer be inverted since preferredTransform handles this. Remember that the asset was never actually transformed yet. newAssetSize just represents the size the video is going to be after you call instruction.setTransform(transform). Since transform is the first transform in concatenation, this is the size that the scale and translate transforms will be using, which is why we needed to reference newAssetSize after applying transform. Also you should concatenate in this order: transform -> scale -> translate, otherwise you won't get desired results
var scaleRatio = UIScreen.MainScreen.Bounds.Height / newAssetSize.Height; //change back to height. Keep in mind that this scaleRatio will fill the height of the screen first and the width will probably exceed the screen bounds. I had it set like this because I was displaying my video in a view that is much smaller than the screen size. If you want to display the video centered on the phone screen, try using scaleRation = UIScreen.MainScreen.Bounds.Width / newAssetSize.Width. This will scale the video to fit the width of the screen perfectly and then the height will be whatever it is with respect to the videos aspect ratio.
//Apply scale to transform. Transform is never actually applied unless you do this.
var _coreGraphic = transform.Scale(scaleRatio, scaleRatio);
var tx = UIScreen.MainScreen.Bounds.Width / 2 - newAssetSize.Height * scaleRatio / 2;
var ty = UIScreen.MainScreen.Bounds.Height / 2 - newAssetSize.Width * scaleRatio / 2;
_coreGraphic.Translate(tx, ty);
instruction.SetTransform(_coreGraphic, currentTime);
}
var endTime = CMTime.Add(currentTime, asset.Duration);
instruction.SetOpacity(0, endTime);
return instruction;
}
Ok, so thanks to Shawn's help I have accomplished what I was trying to do.
There were 2 main mistakes in my code that generated this problem, the first one was how the property of the CMTime given to VideoTrack was set: Start = new CMTime(0,0), instead of Start = new CMTime.Zero,. I still don't know what difference does it make, but it prevented the code from displaying the video and the audio of each asset, leaving a video with the length of all the clips combined and the background of AVMutableVideoCompositionInstruction.
The second mistake was how I set the instructions, the configuration that worked for me can be found in the following code.
Here is the final function working as correctly:
public void MergeClips()
{
//microphone
AVCaptureDevice microphone = AVCaptureDevice.DefaultDeviceWithMediaType(AVMediaType.Audio);
AVMutableComposition mixComposition = AVMutableComposition.Create();
AVVideoCompositionLayerInstruction[] Instruction_Array = new AVVideoCompositionLayerInstruction[Clips.Count];
foreach (string clip in Clips)
{
var asset = AVUrlAsset.FromUrl(new NSUrl(clip, false)) as AVUrlAsset;
#region HoldVideoTrack
//This range applies to the video, not to the mixcomposition
CMTimeRange range = new CMTimeRange()
{
Start = CMTime.Zero,
Duration = asset.Duration
};
var duration = mixComposition.Duration;
NSError error;
AVMutableCompositionTrack videoTrack = mixComposition.AddMutableTrack(AVMediaType.Video, 0);
AVAssetTrack assetVideoTrack = asset.TracksWithMediaType(AVMediaType.Video)[0];
videoTrack.InsertTimeRange(range, assetVideoTrack, duration, out error);
videoTrack.PreferredTransform = assetVideoTrack.PreferredTransform;
if (microphone != null)
{
AVMutableCompositionTrack audioTrack = mixComposition.AddMutableTrack(AVMediaType.Audio, 0);
AVAssetTrack assetAudioTrack = asset.TracksWithMediaType(AVMediaType.Audio)[0];
audioTrack.InsertTimeRange(range, assetAudioTrack, duration, out error);
}
#endregion
#region Instructions
int counter = Clips.IndexOf(clip);
Instruction_Array[counter] = SetInstruction(asset, mixComposition.Duration, videoTrack);
#endregion
}
// 6
AVMutableVideoCompositionInstruction mainInstruction = AVMutableVideoCompositionInstruction.Create() as AVMutableVideoCompositionInstruction;
CMTimeRange rangeIns = new CMTimeRange()
{
Start = new CMTime(0, 0),
Duration = mixComposition.Duration
};
mainInstruction.TimeRange = rangeIns;
mainInstruction.LayerInstructions = Instruction_Array;
var mainComposition = AVMutableVideoComposition.Create();
mainComposition.Instructions = new AVVideoCompositionInstruction[1] { mainInstruction };
mainComposition.FrameDuration = new CMTime(1, 30);
mainComposition.RenderSize = new CGSize(mixComposition.NaturalSize.Height, mixComposition.NaturalSize.Width);
finalVideo_path = NSUrl.FromFilename(Path.Combine(Path.GetTempPath(), "Whole2.mov"));
if (File.Exists(Path.GetTempPath() + "Whole2.mov"))
{
File.Delete(Path.GetTempPath() + "Whole2.mov");
}
//... export video ...
AVAssetExportSession exportSession = new AVAssetExportSession(mixComposition, AVAssetExportSessionPreset.HighestQuality)
{
OutputUrl = NSUrl.FromFilename(Path.Combine(Path.GetTempPath(), "Whole2.mov")),
OutputFileType = AVFileType.QuickTimeMovie,
ShouldOptimizeForNetworkUse = true,
VideoComposition = mainComposition
};
exportSession.ExportAsynchronously(_OnExportDone);
}
private AVMutableVideoCompositionLayerInstruction SetInstruction(AVAsset asset, CMTime currentTime, AVAssetTrack mixComposition_video_Track)
{
var instruction = AVMutableVideoCompositionLayerInstruction.FromAssetTrack(mixComposition_video_Track);
var startTime = CMTime.Subtract(currentTime, asset.Duration);
//NaturalSize.Height is passed as a width parameter because IOS stores the video recording horizontally
CGAffineTransform translateToCenter = CGAffineTransform.MakeTranslation(mixComposition_video_Track.NaturalSize.Height, 0);
//Angle in radiants, not in degrees
CGAffineTransform rotate = CGAffineTransform.Rotate(translateToCenter, (nfloat)(Math.PI / 2));
instruction.SetTransform(rotate, (CMTime.Subtract(currentTime, asset.Duration)));
instruction.SetOpacity(1, startTime);
instruction.SetOpacity(0, currentTime);
return instruction;
}
As I said I solved my problem thanks to Shawn's help, and most of this code was translated to C# from his answers, so please, if you were planning on voting up this answer, vote up Shawn's one instead, or both.
Related
Need help because I'm new to OpenGL.
So, my task is to create a control that will draw in real time the process of cutting a workpiece on a CNC machine.
I tried to do it classically through glBegin, glEnd and everything works fine, but due to the large number of vertices, it starts to work slowly at the end. Therefore, I decided to try using VertexBufferArray () and VertexBuffer () - this also works, but in this situation I do not understand how to change the line width and its type (specifically, I have two types - a regular line and a dash-dotted line).
Here is my method where i use array
private void CreateBufferAndDraw(OpenGL GL)
{
try
{
if (shaderProgram == null && vertexBufferArray == null)
{
var vertexShaderSource = ManifestResourceLoader.LoadTextFile("vertex_shader.glsl");
var fragmentShaderSource = ManifestResourceLoader.LoadTextFile("fragment_shader.glsl");
shaderProgram = new ShaderProgram();
shaderProgram.Create(gL, vertexShaderSource, fragmentShaderSource, null);
attribute_vpos = (uint)gL.GetAttribLocation(shaderProgram.ShaderProgramObject, "vPosition");
attribute_vcol = (uint)gL.GetAttribLocation(shaderProgram.ShaderProgramObject, "vColor");
shaderProgram.AssertValid(gL);
}
if (_data == null)
{
_data = new vec3[_Points.Count];
}
else
{
if (_data.Length != _Points.Count)
{
_data = (vec3[])ResizeArray(_data, _Points.Count);
}
}
if (_dataColor == null)
{
_dataColor = new vec3[_Points.Count];
}
else
{
if (_dataColor.Length != _Points.Count)
{
_dataColor = (vec3[])ResizeArray(_dataColor, _Points.Count);
}
}
for (int i = _dataTail; i < _Points.Count; i++)
{
_data[i].y = _Points[i].Y;
_data[i].x = _Points[i].X;
_data[i].z = _Points[i].Z;
_dataColor[i] = new vec3(_Points[i].ToolColor.R / 255.0f, _Points[i].ToolColor.G / 255.0f, _Points[i].ToolColor.B / 255.0f);
}
_dataTail = _Points.Count;
// Create the vertex array object.
vertexBufferArray = new VertexBufferArray();
vertexBufferArray.Create(GL);
vertexBufferArray.Bind(GL);
// Create a vertex buffer for the vertex data.
var vertexDataBuffer = new VertexBuffer();
vertexDataBuffer.Create(GL);
vertexDataBuffer.Bind(GL);
vertexDataBuffer.SetData(GL, attribute_vpos, _data, false, 3);
// Now do the same for the colour data.
var colourDataBuffer = new VertexBuffer();
colourDataBuffer.Create(GL);
colourDataBuffer.Bind(GL);
colourDataBuffer.SetData(GL, attribute_vcol, _dataColor, false, 3);
// Unbind the vertex array, we've finished specifying data for it.
vertexBufferArray.Unbind(GL);
// Bind the shader, set the matrices.
shaderProgram.Bind(GL);
// Set matrixs for shader program
float rads = (90.0f / 360.0f) * (float)Math.PI * 2.0f;
mat4 _mviewdata = glm.translate(new mat4(1f), new vec3(_track_X, _track_Y, _track_Z));
mat4 _projectionMatrix = glm.perspective(rads, (float)this.trackgl_control.Height / (float)this.trackgl_control.Width, 0.001f, 1000f);
mat4 _modelMatrix = glm.lookAt(new vec3(0, 0, -1), new vec3(0, 0, 0), new vec3(0, -1, -1));
shaderProgram.SetUniformMatrix4(GL, "projectionMatrix", _projectionMatrix.to_array());
shaderProgram.SetUniformMatrix4(GL, "viewMatrix", _modelMatrix.to_array());
shaderProgram.SetUniformMatrix4(GL, "modelMatrix", _mviewdata.to_array());
// Bind the out vertex array.
vertexBufferArray.Bind(GL);
GL.LineWidth(5.0f);
// Draw the square.
GL.DrawArrays(OpenGL.GL_LINE_STRIP, 0, _dataTail);
// Unbind our vertex array and shader.
vertexBufferArray.Unbind(GL);
shaderProgram.Unbind(GL);
}
catch (Exception ex) { MessageBox.Show("CreateAndPlotData" + "\n" + ex.ToString()); }
}
This is what i get
As you see, all lines have the same width. So, my question is: Does anyone know what i can do about this?
And another one: what if i need to show a point of the current location of the instrument? I should create another array with one Vertex ?
p.s. Sry for my english, and here here is picture of what i get with glBegin/glEnd
No tag spamming intended.
I'm doing this project partially guided by tutorials in C ++ and C #. Therefore, if someone knows how to do this in C ++, then in this case I will just try to implement the same in C#.
I'm zxing to estimate the 4 corner points of the QR Code. Following is my code to estimate the corner points.
LuminanceSource source = new RGBLuminanceSource(barcodeBitmap,QRTexture.width, QRTexture.height);
var options = new DecodingOptions { PossibleFormats = new List<BarcodeFormat> { BarcodeFormat.QR_CODE }, TryHarder = true };
this.reader = new BarcodeReader(null, null, ls => new GlobalHistogramBinarizer(ls)) { AutoRotate = false, TryInverted = false, Options = options };
Result result = this.reader.Decode(source);
This gives me a result points which has the four corners of the QR Code. How do I overlay a 3D object over the qr code based on the position of these corner points?
I don't know that QR reader you are using but generally you basically only need 3 points e.g.
A-------B
|
| X
|
C
X is where you want to place your object
So simply at
// Assuming given values
Vector3 A; // top-left corner
Vector3 B; // top-right corner
Vector3 C; // bottom-left corner
GameObject obj;
var vectorAtoB = B - A;
var vectorAtoC = C - A;
obj.transform.position = A + vectorAtoB * 0.5f + vectorAtoC * 0.5f;
and then you also need the orientation for your object. Depending on your needs of course but the easiest way is to set the object's Transform.forward and Transform.right (it is enough to set two axis as the third one will be correct automatically)
var vectorCtoA;
obj.transform.forward = vectorCtoA;
obj.transform.right = vectorAtoB;
If you also need the scale then it gets tricky - or well at least you need one given value more:
// This is the expected QR code edge length
// if the QR code has exactly this size then the model will have scale 1,1,1
// otherwise it is scaled according to the QR code size
float normalSize;
var qrEdgeLength = vectorAtoB.magnitude;
obj.transform.localScale = Vector3.one * qrEdgeLength / normalSize;
I need to build a graphic train schedule visualisation tool in C#. Actually I have to rebuild this perfect tool in C#.
Marey's Trains
The graphs have to be zoomable, scrollable and printable/exportable to PDF with vector graphical elements.
Could you give me some tips? How should I start it? What sort of libraries should I use?
Is it worth to try using graphing libraries like OxyPlot? Maybe it's not the best because of the special axes and irregular grids - as I think. What's your opinion?
No matter which chart tool you use, once you need special types of display you will always have to add some extra coding.
Here is an example of using the MSChart control.
Do note that it has a limitation wrt to exporting vector formats:
It can export to various formats, including 3 EMF types; however only some application can actually use those. Not sure about the PDF libary you use..!
If you can't use the Emf formats you can get nice results by making the Chart control really big, exporting to Png and then making the Dpi resolution much larger the the default screen resolution it has after saving.. Setting it to 600 or 1200dpi should do for most pdf uses..
Now lets look at an example:
A few notes:
I have made my life easier in a number of ways. I have only coded for one direction and I have not reversed the rooster, so it goes only bottom to top.
I have not used real data but made them up.
I have not created one or more classes to hold the station data; instead I use a very simple Tuple.
I have not created a DataTable to hold the train data. Instead I make them up and add them to the chart on the fly..
I didn't test, but zooming and scrolling should work as well..
Here is the List<Tuple> that holds my station data:
// station name, distance, type: 0=terminal, 1=normal, 2=main station
List<Tuple<string, double, int>> TrainStops = null;
Here is how I set up the chart:
Setup24HoursAxis(chart1, DateTime.Today);
TrainStops = SetupTrainStops(17);
SetupTrainStopAxis(chart1);
for (int i = 0; i < 23 * 3; i++)
{
AddTrainStopSeries(chart1, DateTime.Today.Date.AddMinutes(i * 20),
17 - rnd.Next(4), i% 5 == 0 ? 1 : 0);
}
// this exports the image above:
chart1.SaveImage("D:\\trains.png", ChartImageFormat.Png);
This creates one train every 20 minutes with 14-17 stops and every 5th train a fast one.
Here are the routines I call:
Setting up the x-axis for hold one day's worth of data is straightforward.
public static void Setup24HoursAxis(Chart chart, DateTime dt)
{
chart.Legends[0].Enabled = false;
Axis ax = chart.ChartAreas[0].AxisX;
ax.IntervalType = DateTimeIntervalType.Hours;
ax.Interval = 1;
ax.Minimum = dt.ToOADate();
ax.Maximum = (dt.AddHours(24)).ToOADate();
ax.LabelStyle.Format = "H:mm";
}
Creating a List of stations with random distances is also very simple. I made the 1st and last ones terminals and every 5th a main station.
public List<Tuple<string, double, int>> SetupTrainStops(int count)
{
var stops = new List<Tuple<string, double, int>>();
Random rnd = new Random(count);
for (int i = 0; i < count; i++)
{
string n = (char)(i+(byte)'A') + "-Street";
double d = 1 + rnd.Next(3) + rnd.Next(4) + rnd.Next(5) / 10d;
if (d < 3) d = 3; // a minimum distance so the label won't touch
int t = (i == 0 | i == count-1) ? 0 : rnd.Next(5)==0 ? 2 : 1;
var ts = new Tuple<string, double, int>(n, d, t);
stops.Add(ts);
}
return stops;
}
Now that we have the train stops we can set up the y-axis:
public void SetupTrainStopAxis(Chart chart)
{
Axis ay = chart.ChartAreas[0].AxisY;
ay.LabelStyle.Font = new Font("Consolas", 8f);
double totalDist = 0;
for (int i = 0; i < TrainStops.Count; i++)
{
CustomLabel cl = new CustomLabel();
cl.Text = TrainStops[i].Item1;
cl.FromPosition = totalDist - 0.1d;
cl.ToPosition = totalDist + 0.1d;
totalDist += TrainStops[i].Item2;
cl.ForeColor = TrainStops[i].Item3 == 1 ? Color.DimGray : Color.Black;
ay.CustomLabels.Add(cl);
}
ay.Minimum = 0;
ay.Maximum = totalDist;
ay.MajorGrid.Enabled = false;
ay.MajorTickMark.Enabled = false;
}
A few notes are called for here:
As the values are quite dynamic we can't use normal Labels which would come with the fixed Interval spacing.
So we create CustomLabels instead.
For these we need two values to determine the space into which they shall be centered. So we create a small span by adding/subtracting 0.1d.
We have calculated the total distance and use it to set up the Maximum of the y-axis. Again: To mimick the schedule you show you will have to do some reversing here and there..
By adding CustomLabels the normal ones are turned off automatically. As we need MajorGridlines at the irregular intervals we also turn the normal ones off. Hence we must draw them ourselves. Not really hard as you can see..:
For this we code one of the xxxPaint events:
private void chart1_PostPaint(object sender, ChartPaintEventArgs e)
{
Axis ay = chart1.ChartAreas[0].AxisY;
Axis ax = chart1.ChartAreas[0].AxisX;
int x0 = (int) ax.ValueToPixelPosition(ax.Minimum);
int x1 = (int) ax.ValueToPixelPosition(ax.Maximum);
double totalDist = 0;
foreach (var ts in TrainStops)
{
int y = (int)ay.ValueToPixelPosition(totalDist);
totalDist += ts.Item2;
using (Pen p = new Pen(ts.Item3 == 1 ? Color.DarkGray : Color.Black,
ts.Item3 == 1 ? 0.5f : 1f))
e.ChartGraphics.Graphics.DrawLine(p, x0 + 1, y, x1, y);
}
// ** Insert marker drawing code (from update below) here !
}
Note the use of the ValueToPixelPosition conversion functions of the axes!
Now for the last part: How to add a Series of train data..:
public void AddTrainStopSeries(Chart chart, DateTime start, int count, int speed)
{
Series s = chart.Series.Add(start.ToShortTimeString());
s.ChartType = SeriesChartType.Line;
s.Color = speed == 0 ? Color.Black : Color.Brown;
s.MarkerStyle = MarkerStyle.Circle;
s.MarkerSize = 4;
double totalDist = 0;
DateTime ct = start;
for (int i = 0; i < count; i++)
{
var ts = TrainStops[i];
ct = ct.AddMinutes(ts.Item2 * (speed == 0 ? 1 : 1.1d));
DataPoint dp = new DataPoint( ct.ToOADate(), totalDist );
totalDist += TrainStops[i].Item2;
s.Points.Add(dp);
}
}
Note that since my data don't contain real arrival/departure times I calculated them from the distance and some speed factor. You, of course, would use your data!
Also note that I have used a Line chart with extra Marker circles.
Also note that each train series can easily be disabled/hidden or brought back again.
Let's show only the fast trains:
private void cbx_ShowOnlyFastTrains_CheckedChanged(object sender, EventArgs e)
{
foreach (Series s in chart1.Series)
s.Enabled = !cbx_ShowOnlyFastTrains.Checked || s.Color == Color.Brown;
}
Of course for a robust application you will not rely ona magic color ;-)
Instead you could add a Tag object to the Series to hold all sorts of train info.
Update: As you noticed the drawn GridLines cover the Markers. You can insert this piece of code here (**); it will owner-draw the Markers at the end of the xxxPaint event.
int w = chart1.Series[0].MarkerSize;
foreach(Series s in chart1.Series)
foreach(DataPoint dp in s.Points)
{
int x = (int) ax.ValueToPixelPosition(dp.XValue) - w / 2;
int y = (int) ay.ValueToPixelPosition(dp.YValues[0])- w / 2;
using (SolidBrush b = new SolidBrush(dp.Color))
e.ChartGraphics.Graphics.FillEllipse(b, x, y, w, w);
}
Close-up:
I've recently started coding a game in XNA Game Studio using C#, however I am having trouble detecting when a ball hits a particular block, could someone help.
Note: blocksPosition is an Array, as there is multiple of them
Here is my code:
public Boolean DetectBallBox(int BoxNum)
{
if (blocksPosition[BoxNum].X >= ballPosition.X + ball.Width && blocksPosition[BoxNum].Y >= ballPosition.Y + ball.Height)
{
return true;
}
else
{
return false;
}
}
public void detectBallCollision()
{
for (int blockCount = 0; blockCount < 40; blockCount++)
{
if (DetectBallBox(blockCount))
{
Console.WriteLine("Block Hit");
}
}
}
Simple solution:
use Rectangle.Intersects:
Rectangle x = new Rectangle(ball.x, ball.y, ball.width, ball.height);
Rectangle y = new Rectangle(x,y, width, height);
return x.Intersects(y);
Thats really simple answer, becouse ball is more like ellipse... :)
Better solution
Finding intersection between Rectangle and Ellipse:
System.Drawing.Rectangle brick = new Rectangle();
int ballX, ballY, radiusX, radiusY;
brick.Width = // dont see it in your code
brick.Height = // dont see it in your code
brick.X = blocksPosition[BoxNum].X;
brick.Y = blocksPosition[BoxNum].Y;
ballX = ballPosition.X;
ballY = ballPosition.Y;
radiusX = ball.Width;
radiusY = ball.Height;
// find closest point:
var closestX = Math.Min(brick.Width + brick.X, Math.Max(ballX, brick.X));
var closestY = Math.Min(brick.Height + brick.Height, Math.Max(ballY, brick.Y));
// find distance:
var distanceX = ballX - closestX;
var distanceY = ballY - closestY;
// if distance is lesser then circle radius, then we have intersaction:
var dSq = ((distanceX * distanceX) + (distanceY * distanceY))
if (dSq < (radiusX*radiusX))
{
return dSq < (radiusY*radiusY);
}
return false;
The Best solution (©®™):
Use any 2d physics engine, like, lets say Farseer. Its more to learn from start, but in end, it will be much better and easier for you.
Quick and Dirty example of Ball hitting brick will looks like this:
World world = new World(new Vector2(0f, 0f)); // no gravity
Body myBall = world.CreateBody();
myBall.BodyType = BodyType.Dynamic;
CircleShape circleShape = new CircleShape(0.5f);
Fixture fixture = myBall.CreateFixture(circleShape);
Fixture brick = FixtureFactory.CreateRectangle();
// etc... Setting Position and so on.
// Its big topic to discuss here whole project.
and every frame:
world.Step(0.033333f); // 60 fps
In final project, you will have two "spaces" - one physical (where you/farseer will count collisions, forces etc) and second one, where you draw your bodies from physic world (btw - Body is only point in space, Shape is any shape / polygon and Fixture is what is holding Body and Shape together... As i say, its really big topic. Feel free to ask any other question about Farseer :) )
I have a Kinect WPF Application that takes images from the Kinect, does some feature detection using EmguCV (A C# opencv wrapper) and displays the output on the using a WPF image.
I have had this working before, but the application now refuses to update the screen image when the imagesource is written to, but I have not changed the way it works.
the Image(called video) is written to as such:
video.Source = bitmapsource;
in the colorframeready event handler.
This works fine until I introduce some opencv code before the imagesource is written to. It does not matter what source is used, so I don't think it is a conflict there. I have narrowed down the offending EmguCV code to this line:
RecentKeyPoints = surfCPU.DetectKeyPointsRaw(ImageRecent, null);
which jumps straight into the opencv code. It is worth noting that:
ImageRecent has completely different origins to the bitmapsource updating the screen.
Reading video.Source returns the bitmapsource, so it seems to be writing correctly, just not updating the screen.
Let me know if you want any more information...
void nui_ColorFrameReady(object sender, ColorImageFrameReadyEventArgs e)
{
// Checks for a recent Depth Image
if (!TrackingReady) return;
// Stores image
using (ColorImageFrame colorImageFrame = e.OpenColorImageFrame())
{
if (colorImageFrame != null)
{
if (FeatureTracker.ColourImageRecent == null)
//allocate the first time
FeatureTracker.ColourImageRecent = new byte[colorImageFrame.PixelDataLength];
colorImageFrame.CopyPixelDataTo(FeatureTracker.ColourImageRecent);
}
else return;
}
FeatureTracker.FeatureDetect(nui);
//video.Source = FeatureTracker.ColourImageRecent.ToBitmapSource();
video.Source = ((Bitmap)Bitmap.FromFile("test1.png")).ToBitmapSource();
TrackingReady = false;
}
public Bitmap FeatureDetect(KinectSensor nui)
{
byte[] ColourClone = new byte[ColourImageRecent.Length];
Array.Copy(ColourImageRecent, ColourClone, ColourImageRecent.Length);
Bitmap test = (Bitmap)Bitmap.FromFile("test1.png");
test.RotateFlip(RotateFlipType.RotateNoneFlipY);
Image<Gray, Byte> ImageRecent = new Image<Gray, byte>(test);
SURFDetector surfCPU = new SURFDetector(2000, false);
VectorOfKeyPoint RecentKeyPoints;
Matrix<int> indices;
Matrix<float> dist;
Matrix<byte> mask;
bool MatchFailed = false;
// extract SURF features from the object image
RecentKeyPoints = surfCPU.DetectKeyPointsRaw(ImageRecent, null);
//Matrix<float> RecentDescriptors = surfCPU.ComputeDescriptorsRaw(ImageRecent, null, RecentKeyPoints);
//MKeyPoint[] RecentPoints = RecentKeyPoints.ToArray();
// don't feature detect on first attempt, just store image details for next attempt
#region
/*
if (KeyPointsOld == null)
{
KeyPointsOld = RecentKeyPoints;
PointsOld = RecentPoints;
DescriptorsOld = RecentDescriptors;
return ImageRecent.ToBitmap();
}
*/
#endregion
// Attempt to match points to their nearest neighbour
#region
/*
BruteForceMatcher SURFmatcher = new BruteForceMatcher(BruteForceMatcher.DistanceType.L2F32);
SURFmatcher.Add(RecentDescriptors);
int k = 5;
indices = new Matrix<int>(DescriptorsOld.Rows, k);
dist = new Matrix<float>(DescriptorsOld.Rows, k);
*/
// Match features, provide the top k matches
//SURFmatcher.KnnMatch(DescriptorsOld, indices, dist, k, null);
// Create mask and set to allow all features
//mask = new Matrix<byte>(dist.Rows, 1);
//mask.SetValue(255);
#endregion
//Features2DTracker.VoteForUniqueness(dist, 0.8, mask);
// Check number of good maches and for error and end matching if true
#region
//int nonZeroCount = CvInvoke.cvCountNonZero(mask);
//if (nonZeroCount < 5) MatchFailed = true;
/*
try
{
nonZeroCount = Features2DTracker.VoteForSizeAndOrientation(RecentKeyPoints, KeyPointsOld, indices, mask, 1.5, 20);
}
catch (SystemException)
{
MatchFailed = true;
}
if (nonZeroCount < 5) MatchFailed = true;
if (MatchFailed)
{
return ImageRecent.ToBitmap();
}
*/
#endregion
//DepthMapColourCoordsRecent = CreateDepthMap(nui, DepthImageRecent);
//PointDist[] FeatureDistances = DistanceToFeature(indices, mask, RecentPoints);
//Image<Rgb,Byte> rgbimage = ImageRecent.Convert<Rgb, Byte>();
//rgbimage = DrawPoints(FeatureDistances, rgbimage);
// Store recent image data for next feature detect.
//KeyPointsOld = RecentKeyPoints;
//PointsOld = RecentPoints;
//DescriptorsOld = RecentDescriptors;
//CreateDepthMap(nui, iva);
//rgbimage = CreateDepthImage(DepthMapColourCoordsRecent, rgbimage);
// Convert image back to a bitmap
count++;
//Bitmap bitmap3 = rgbimage.ToBitmap();
//bitmapstore = bitmap3;
//bitmap3.Save("test" + count.ToString() + ".png");
return null;
}
This is a little late, but I had a similar problem and thought I'd share my solution.
In my case I was processing the depth stream. The default resolution was 640x480, and Emgu just wasn't able to process the image fast enough to keep up with the frameready handler. As soon as I reduced the depth stream resolution to 320x240 the problem went away.
I also went a bit further and moved my image processing to a different thread which sped it up even more (do a search for ComponentDispatcher.ThreadIdle). I'm still not able to do 640x480 at a reasonable frame rate, but at least the image renders so I can see what's going on.